LLVM 22.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
78 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
79 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
80 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
82 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
84 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
86 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
88 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
90 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
91 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
93 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
94 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
96 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
98 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
100 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
101 case ISD::STRICT_FEXP:
102 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
104 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
105 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
107 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
108 case ISD::STRICT_FLOG:
109 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
111 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
113 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
114 case ISD::STRICT_FMA:
115 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
116 case ISD::STRICT_FMUL:
117 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
119 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
120 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
122 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
124 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
125 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
126 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
127 case ISD::STRICT_FPOW:
128 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
130 case ISD::FPOWI:
131 case ISD::FLDEXP:
132 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
133 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
134 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
135 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
136 case ISD::STRICT_FREM:
137 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
139 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
141 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
143 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
144 case ISD::STRICT_FSIN:
145 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
147 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
149 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
150 case ISD::STRICT_FSUB:
151 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
152 case ISD::STRICT_FTAN:
153 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
155 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
157 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
158 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
159 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
160 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
161 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
162 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
163 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
166 case ISD::SINT_TO_FP:
167 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
168 case ISD::POISON:
169 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
170 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
171 case ISD::AssertNoFPClass: R = GetSoftenedFloat(N->getOperand(0)); break;
177 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
179 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
180 // clang-format on
181 }
182
183 // If R is null, the sub-method took care of registering the result.
184 if (R.getNode()) {
185 assert(R.getNode() != N);
186 SetSoftenedFloat(SDValue(N, ResNo), R);
187 }
188}
189
190SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
191 bool IsStrict = N->isStrictFPOpcode();
192 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
193 unsigned Offset = IsStrict ? 1 : 0;
194 assert(N->getNumOperands() == (1 + Offset) &&
195 "Unexpected number of operands!");
196 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
197 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
199 EVT OpVT = N->getOperand(0 + Offset).getValueType();
200 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
210 bool IsStrict = N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
212 unsigned Offset = IsStrict ? 1 : 0;
213 assert(N->getNumOperands() == (2 + Offset) &&
214 "Unexpected number of operands!");
215 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
216 GetSoftenedFloat(N->getOperand(1 + Offset)) };
217 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
219 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
220 N->getOperand(1 + Offset).getValueType() };
221 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
222 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
223 CallOptions, SDLoc(N),
224 Chain);
225 if (IsStrict)
226 ReplaceValueWith(SDValue(N, 1), Tmp.second);
227 return Tmp.first;
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
231 return BitConvertToInteger(N->getOperand(0));
232}
233
234SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
235 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
236 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
237 GetSoftenedFloat(N->getOperand(0)));
238}
239
240SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
241 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
242 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
243 GetSoftenedFloat(N->getOperand(0)));
244 return NewFence;
245}
246
247SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
248 unsigned ResNo) {
249 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
250 return BitConvertToInteger(Op);
251}
252
253SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
254 // Convert the inputs to integers, and build a new pair out of them.
255 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
257 N->getValueType(0)),
258 BitConvertToInteger(N->getOperand(0)),
259 BitConvertToInteger(N->getOperand(1)));
260}
261
262SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
263 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
264 // In ppcf128, the high 64 bits are always first in memory regardless
265 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
266 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
267 // way. However, APInt's are serialized in an Endian-sensitive fashion,
268 // so on big-Endian targets, the two doubles are output in the wrong
269 // order. Fix this by manually flipping the order of the high 64 bits
270 // and the low 64 bits here.
271 if (DAG.getDataLayout().isBigEndian() &&
272 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
273 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
275 APInt Val(128, words);
276 return DAG.getConstant(Val, SDLoc(CN),
278 CN->getValueType(0)));
279 } else {
280 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
282 CN->getValueType(0)));
283 }
284}
285
286SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
287 SDValue Src = N->getOperand(0);
288 assert(Src.getValueType() == MVT::ppcf128 &&
289 "In floats only ppcf128 can be extracted by element!");
290 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
291 N->getValueType(0).changeTypeToInteger(),
292 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
293}
294
295SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
296 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
299 NewOp, N->getOperand(1));
300}
301
302SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
303 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
304 unsigned Size = NVT.getSizeInBits();
305
306 // Mask = ~(1 << (Size-1))
308 API.clearBit(Size - 1);
309 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
310 SDValue Op = GetSoftenedFloat(N->getOperand(0));
311 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
312}
313
314SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
315 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
316 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
317 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
318 RTLIB::FMIN_F32,
319 RTLIB::FMIN_F64,
320 RTLIB::FMIN_F80,
321 RTLIB::FMIN_F128,
322 RTLIB::FMIN_PPCF128));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
326 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
327 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
328 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
329 RTLIB::FMAX_F32,
330 RTLIB::FMAX_F64,
331 RTLIB::FMAX_F80,
332 RTLIB::FMAX_F128,
333 RTLIB::FMAX_PPCF128));
334}
335
336SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
337 return SoftenFloatRes_Binary(
338 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
339 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
340 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128));
341}
342
343SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
344 return SoftenFloatRes_Binary(
345 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
346 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
347 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128));
348}
349
350SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
351 return SoftenFloatRes_Binary(
352 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_F32,
353 RTLIB::FMINIMUM_F64, RTLIB::FMINIMUM_F80,
354 RTLIB::FMINIMUM_F128, RTLIB::FMINIMUM_PPCF128));
355}
356
357SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
358 return SoftenFloatRes_Binary(
359 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_F32,
360 RTLIB::FMAXIMUM_F64, RTLIB::FMAXIMUM_F80,
361 RTLIB::FMAXIMUM_F128, RTLIB::FMAXIMUM_PPCF128));
362}
363
364SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
365 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
366 RTLIB::ADD_F32,
367 RTLIB::ADD_F64,
368 RTLIB::ADD_F80,
369 RTLIB::ADD_F128,
370 RTLIB::ADD_PPCF128));
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
374 return SoftenFloatRes_Unary(
375 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
376 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
377}
378
379SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
380 return SoftenFloatRes_Unary(
381 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
382 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
383}
384
385SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
386 return SoftenFloatRes_Unary(
387 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
388 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
392 return SoftenFloatRes_Binary(
393 N,
394 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32, RTLIB::ATAN2_F64,
395 RTLIB::ATAN2_F80, RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
399 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
400 RTLIB::CBRT_F32,
401 RTLIB::CBRT_F64,
402 RTLIB::CBRT_F80,
403 RTLIB::CBRT_F128,
404 RTLIB::CBRT_PPCF128));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
408 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
409 RTLIB::CEIL_F32,
410 RTLIB::CEIL_F64,
411 RTLIB::CEIL_F80,
412 RTLIB::CEIL_F128,
413 RTLIB::CEIL_PPCF128));
414}
415
416SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
417 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
418 SDValue RHS = BitConvertToInteger(N->getOperand(1));
419 SDLoc dl(N);
420
421 EVT LVT = LHS.getValueType();
422 EVT RVT = RHS.getValueType();
423
424 unsigned LSize = LVT.getSizeInBits();
425 unsigned RSize = RVT.getSizeInBits();
426
427 // First get the sign bit of second operand.
428 SDValue SignBit = DAG.getNode(
429 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
430 DAG.getConstant(RSize - 1, dl,
431 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
432 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
433
434 // Shift right or sign-extend it if the two operands have different types.
435 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
436 if (SizeDiff > 0) {
437 SignBit =
438 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
439 DAG.getConstant(SizeDiff, dl,
440 TLI.getShiftAmountTy(SignBit.getValueType(),
441 DAG.getDataLayout())));
442 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
443 } else if (SizeDiff < 0) {
444 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
445 SignBit =
446 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
447 DAG.getConstant(-SizeDiff, dl,
448 TLI.getShiftAmountTy(SignBit.getValueType(),
449 DAG.getDataLayout())));
450 }
451
452 // Clear the sign bit of the first operand.
453 SDValue Mask = DAG.getNode(
454 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
455 DAG.getConstant(LSize - 1, dl,
456 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
457 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
458 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
459
460 // Or the value with the sign bit.
461 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
462}
463
464SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
465 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
466 RTLIB::COS_F32,
467 RTLIB::COS_F64,
468 RTLIB::COS_F80,
469 RTLIB::COS_F128,
470 RTLIB::COS_PPCF128));
471}
472
473SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
474 return SoftenFloatRes_Unary(
475 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
476 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
477}
478
479SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
480 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
481 RTLIB::DIV_F32,
482 RTLIB::DIV_F64,
483 RTLIB::DIV_F80,
484 RTLIB::DIV_F128,
485 RTLIB::DIV_PPCF128));
486}
487
488SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
489 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
490 RTLIB::EXP_F32,
491 RTLIB::EXP_F64,
492 RTLIB::EXP_F80,
493 RTLIB::EXP_F128,
494 RTLIB::EXP_PPCF128));
495}
496
497SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
498 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
499 RTLIB::EXP2_F32,
500 RTLIB::EXP2_F64,
501 RTLIB::EXP2_F80,
502 RTLIB::EXP2_F128,
503 RTLIB::EXP2_PPCF128));
504}
505
506SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
507 return SoftenFloatRes_Unary(
508 N,
509 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
510 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
511}
512
513SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
514 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
515 RTLIB::FLOOR_F32,
516 RTLIB::FLOOR_F64,
517 RTLIB::FLOOR_F80,
518 RTLIB::FLOOR_F128,
519 RTLIB::FLOOR_PPCF128));
520}
521
522SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
523 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
524 RTLIB::LOG_F32,
525 RTLIB::LOG_F64,
526 RTLIB::LOG_F80,
527 RTLIB::LOG_F128,
528 RTLIB::LOG_PPCF128));
529}
530
531SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
532 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
533 RTLIB::LOG2_F32,
534 RTLIB::LOG2_F64,
535 RTLIB::LOG2_F80,
536 RTLIB::LOG2_F128,
537 RTLIB::LOG2_PPCF128));
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
541 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
542 RTLIB::LOG10_F32,
543 RTLIB::LOG10_F64,
544 RTLIB::LOG10_F80,
545 RTLIB::LOG10_F128,
546 RTLIB::LOG10_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
550 bool IsStrict = N->isStrictFPOpcode();
551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
552 unsigned Offset = IsStrict ? 1 : 0;
553 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
554 GetSoftenedFloat(N->getOperand(1 + Offset)),
555 GetSoftenedFloat(N->getOperand(2 + Offset)) };
556 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
558 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
559 N->getOperand(1 + Offset).getValueType(),
560 N->getOperand(2 + Offset).getValueType() };
561 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
563 GetFPLibCall(N->getValueType(0),
564 RTLIB::FMA_F32,
565 RTLIB::FMA_F64,
566 RTLIB::FMA_F80,
567 RTLIB::FMA_F128,
568 RTLIB::FMA_PPCF128),
569 NVT, Ops, CallOptions, SDLoc(N), Chain);
570 if (IsStrict)
571 ReplaceValueWith(SDValue(N, 1), Tmp.second);
572 return Tmp.first;
573}
574
575SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
576 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
577 RTLIB::MUL_F32,
578 RTLIB::MUL_F64,
579 RTLIB::MUL_F80,
580 RTLIB::MUL_F128,
581 RTLIB::MUL_PPCF128));
582}
583
584SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
585 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
586 RTLIB::NEARBYINT_F32,
587 RTLIB::NEARBYINT_F64,
588 RTLIB::NEARBYINT_F80,
589 RTLIB::NEARBYINT_F128,
590 RTLIB::NEARBYINT_PPCF128));
591}
592
593SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
595 SDLoc dl(N);
596
597 // Expand Y = FNEG(X) -> Y = X ^ sign mask
598 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
599 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
600 DAG.getConstant(SignMask, dl, NVT));
601}
602
603SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
604 bool IsStrict = N->isStrictFPOpcode();
605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
606 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
607
608 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
609
610 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
611 Op = GetPromotedFloat(Op);
612 // If the promotion did the FP_EXTEND to the destination type for us,
613 // there's nothing left to do here.
614 if (Op.getValueType() == N->getValueType(0)) {
615 if (IsStrict)
616 ReplaceValueWith(SDValue(N, 1), Chain);
617 return BitConvertToInteger(Op);
618 }
619 }
620
621 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
622 // -> f32, so proceed in two stages. Also, it's entirely possible for both
623 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
624 // than FP16_TO_FP.
625 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
626 N->getValueType(0) != MVT::f32) {
627 if (IsStrict) {
629 { MVT::f32, MVT::Other }, { Chain, Op });
630 Chain = Op.getValue(1);
631 } else {
632 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
633 }
634 }
635
636 if (Op.getValueType() == MVT::bf16) {
637 // FIXME: Need ReplaceValueWith on chain in strict case
638 return SoftenFloatRes_BF16_TO_FP(N);
639 }
640
641 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
644 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
645 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
646 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
647 CallOptions, SDLoc(N),
648 Chain);
649 if (IsStrict)
650 ReplaceValueWith(SDValue(N, 1), Tmp.second);
651 return Tmp.first;
652}
653
654// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
655// nodes?
656SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
657 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
658 SDValue Op = N->getOperand(0);
660 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
661 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
662 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
663 CallOptions, SDLoc(N)).first;
664 if (N->getValueType(0) == MVT::f32)
665 return Res32;
666
667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
668 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
670 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
671}
672
673// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
674// nodes?
675SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
676 assert(N->getValueType(0) == MVT::f32 &&
677 "Can only soften BF16_TO_FP with f32 result");
678 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
679 SDValue Op = N->getOperand(0);
680 SDLoc DL(N);
681 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
682 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
683 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
684 DAG.getShiftAmountConstant(16, NVT, DL));
685 return Res;
686}
687
688SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
689 bool IsStrict = N->isStrictFPOpcode();
690 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
691 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
692 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
693 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
694 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
696 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
697 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
698 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
699 CallOptions, SDLoc(N),
700 Chain);
701 if (IsStrict)
702 ReplaceValueWith(SDValue(N, 1), Tmp.second);
703 return Tmp.first;
704}
705
706SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
707 return SoftenFloatRes_Binary(N, RTLIB::getPOW(N->getValueType(0)));
708}
709
710SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
711 bool IsStrict = N->isStrictFPOpcode();
712 unsigned Offset = IsStrict ? 1 : 0;
713 bool IsPowI =
714 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
715 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
716
717 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
718 : RTLIB::getLDEXP(N->getValueType(0));
719 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
720 if (!TLI.getLibcallName(LC)) {
721 // Some targets don't have a powi libcall; use pow instead.
722 // FIXME: Implement this if some target needs it.
723 DAG.getContext()->emitError("do not know how to soften fpowi to fpow");
724 if (IsStrict)
725 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
726 return DAG.getPOISON(NVT);
727 }
728
729 if (DAG.getLibInfo().getIntSize() !=
730 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
731 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
732 // would use the wrong type for the argument.
733 DAG.getContext()->emitError("powi exponent does not match sizeof(int)");
734 if (IsStrict)
735 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
736 return DAG.getPOISON(NVT);
737 }
738
739 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
740 N->getOperand(1 + Offset) };
741 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
743 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
744 N->getOperand(1 + Offset).getValueType() };
745 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
746 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
747 CallOptions, SDLoc(N),
748 Chain);
749 if (IsStrict)
750 ReplaceValueWith(SDValue(N, 1), Tmp.second);
751 return Tmp.first;
752}
753
754SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
755 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
756 EVT VT0 = N->getValueType(0);
757 EVT VT1 = N->getValueType(1);
758 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
759 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
760 SDLoc DL(N);
761
762 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
763 // If the exponent does not match with sizeof(int) a libcall would use the
764 // wrong type for the argument.
765 // TODO: Should be able to handle mismatches.
766 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
767 SDValue PoisonExp = DAG.getPOISON(VT1);
768 ReplaceValueWith(SDValue(N, 1), PoisonExp);
769 return DAG.getMergeValues({DAG.getPOISON(NVT0), PoisonExp}, DL);
770 }
771
772 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
773
776 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
777 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
778 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
779
780 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
781 // but we only really need to handle the 0th one for softening anyway.
782 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0)
783 .setOpsTypeOverrides(CallOpsTypeOverrides);
784
785 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
786 /*Chain=*/SDValue());
787 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
788 auto PtrInfo =
790
791 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
792
793 ReplaceValueWith(SDValue(N, 1), LoadExp);
794 return ReturnVal;
795}
796
797bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
798 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
799 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
800 EVT VT = N->getValueType(0);
801
802 assert(VT == N->getValueType(1) &&
803 "expected both return values to have the same type");
804
805 if (!TLI.getLibcallName(LC))
806 return false;
807
808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
809
810 SDLoc DL(N);
811
812 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(N->getOperand(0))};
813 SmallVector<EVT, 3> OpsVT = {VT};
814
815 std::array<SDValue, 2> StackSlots;
816 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
818 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
819 if (ResNum == CallRetResNo)
820 continue;
821 SDValue StackSlot = DAG.CreateStackTemporary(NVT);
822 Ops.push_back(StackSlot);
823 OpsVT.push_back(StackSlot.getValueType());
824 StackSlots[ResNum] = StackSlot;
825 CallOpsTypeOverrides.push_back(PointerTy);
826 }
827
829 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
830 // but since both returns have the same type it should be okay.
831 CallOptions.setTypeListBeforeSoften({OpsVT}, VT)
832 .setOpsTypeOverrides(CallOpsTypeOverrides);
833
834 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL,
835 /*Chain=*/SDValue());
836
837 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
838 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
839 auto PtrInfo =
841 return DAG.getLoad(NVT, DL, Chain, StackSlot, PtrInfo);
842 };
843
844 for (auto [ResNum, SlackSlot] : enumerate(StackSlots)) {
845 if (CallRetResNo == ResNum) {
846 SetSoftenedFloat(SDValue(N, ResNum), ReturnVal);
847 continue;
848 }
849 SetSoftenedFloat(SDValue(N, ResNum), CreateStackLoad(SlackSlot));
850 }
851
852 return true;
853}
854
855SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
856 EVT VT = N->getValueType(0);
857 if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getSINCOS(VT)))
858 return SDValue();
859
860 // Fall back on softening the separate sin and cos calls if available.
861 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
862 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
863
864 SDValue SoftSin, SoftCos;
865 if (!TLI.getLibcallName(SinLC) || !TLI.getLibcallName(CosLC)) {
866 DAG.getContext()->emitError("do not know how to soften fsincos");
867
868 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
869 SoftSin = SoftCos = DAG.getPOISON(NVT);
870 } else {
871 SoftSin = SoftenFloatRes_Unary(N, SinLC);
872 SoftCos = SoftenFloatRes_Unary(N, CosLC);
873 }
874
875 SetSoftenedFloat(SDValue(N, 0), SoftSin);
876 SetSoftenedFloat(SDValue(N, 1), SoftCos);
877 return SDValue();
878}
879
880SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
881 EVT VT = N->getValueType(0);
882 if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(VT),
883 /*CallRetResNo=*/0))
884 return SDValue();
885
886 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
887 DAG.getContext()->emitError("do not know how to soften fmodf");
888 SDValue Poison = DAG.getPOISON(NVT);
889 SetSoftenedFloat(SDValue(N, 0), Poison);
890 SetSoftenedFloat(SDValue(N, 1), Poison);
891 return SDValue();
892}
893
894SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
895 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
896 RTLIB::REM_F32,
897 RTLIB::REM_F64,
898 RTLIB::REM_F80,
899 RTLIB::REM_F128,
900 RTLIB::REM_PPCF128));
901}
902
903SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
904 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
905 RTLIB::RINT_F32,
906 RTLIB::RINT_F64,
907 RTLIB::RINT_F80,
908 RTLIB::RINT_F128,
909 RTLIB::RINT_PPCF128));
910}
911
912SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
913 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
914 RTLIB::ROUND_F32,
915 RTLIB::ROUND_F64,
916 RTLIB::ROUND_F80,
917 RTLIB::ROUND_F128,
918 RTLIB::ROUND_PPCF128));
919}
920
921SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
922 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
923 RTLIB::ROUNDEVEN_F32,
924 RTLIB::ROUNDEVEN_F64,
925 RTLIB::ROUNDEVEN_F80,
926 RTLIB::ROUNDEVEN_F128,
927 RTLIB::ROUNDEVEN_PPCF128));
928}
929
930SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
931 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
932 RTLIB::SIN_F32,
933 RTLIB::SIN_F64,
934 RTLIB::SIN_F80,
935 RTLIB::SIN_F128,
936 RTLIB::SIN_PPCF128));
937}
938
939SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
940 return SoftenFloatRes_Unary(
941 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
942 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
943}
944
945SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
946 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
947 RTLIB::SQRT_F32,
948 RTLIB::SQRT_F64,
949 RTLIB::SQRT_F80,
950 RTLIB::SQRT_F128,
951 RTLIB::SQRT_PPCF128));
952}
953
954SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
955 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
956 RTLIB::SUB_F32,
957 RTLIB::SUB_F64,
958 RTLIB::SUB_F80,
959 RTLIB::SUB_F128,
960 RTLIB::SUB_PPCF128));
961}
962
963SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
964 return SoftenFloatRes_Unary(
965 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
966 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
967}
968
969SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
970 return SoftenFloatRes_Unary(
971 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
972 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
973}
974
975SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
976 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
977 RTLIB::TRUNC_F32,
978 RTLIB::TRUNC_F64,
979 RTLIB::TRUNC_F80,
980 RTLIB::TRUNC_F128,
981 RTLIB::TRUNC_PPCF128));
982}
983
984SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
985 LoadSDNode *L = cast<LoadSDNode>(N);
986 EVT VT = N->getValueType(0);
987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
988 SDLoc dl(N);
989
990 auto MMOFlags =
991 L->getMemOperand()->getFlags() &
993 SDValue NewL;
994 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
995 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
996 L->getChain(), L->getBasePtr(), L->getOffset(),
997 L->getPointerInfo(), NVT, L->getBaseAlign(), MMOFlags,
998 L->getAAInfo());
999 // Legalized the chain result - switch anything that used the old chain to
1000 // use the new one.
1001 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1002 return NewL;
1003 }
1004
1005 // Do a non-extending load followed by FP_EXTEND.
1006 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
1007 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
1008 L->getPointerInfo(), L->getMemoryVT(), L->getBaseAlign(),
1009 MMOFlags, L->getAAInfo());
1010 // Legalized the chain result - switch anything that used the old chain to
1011 // use the new one.
1012 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1013 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
1014 return BitConvertToInteger(ExtendNode);
1015}
1016
1017SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
1018 AtomicSDNode *L = cast<AtomicSDNode>(N);
1019 EVT VT = N->getValueType(0);
1020 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1021 SDLoc dl(N);
1022
1023 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1024 SDValue NewL =
1025 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
1026 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
1027
1028 // Legalized the chain result - switch anything that used the old chain to
1029 // use the new one.
1030 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1031 return NewL;
1032 }
1033
1034 report_fatal_error("softening fp extending atomic load not handled");
1035}
1036
1037SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
1038 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
1039 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
1040 return DAG.getSelect(SDLoc(N),
1041 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1042}
1043
1044SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
1045 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
1046 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
1047 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1048 LHS.getValueType(), N->getOperand(0),
1049 N->getOperand(1), LHS, RHS, N->getOperand(4));
1050}
1051
1052SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
1053 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1054 N->getValueType(0)));
1055}
1056
1057SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
1058 SDValue Chain = N->getOperand(0); // Get the chain.
1059 SDValue Ptr = N->getOperand(1); // Get the pointer.
1060 EVT VT = N->getValueType(0);
1061 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1062 SDLoc dl(N);
1063
1064 SDValue NewVAARG;
1065 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
1066 N->getConstantOperandVal(3));
1067
1068 // Legalized the chain result - switch anything that used the old chain to
1069 // use the new one.
1070 if (N != NewVAARG.getValue(1).getNode())
1071 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
1072 return NewVAARG;
1073}
1074
1075SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1076 bool IsStrict = N->isStrictFPOpcode();
1077 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1078 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1079 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
1080 EVT RVT = N->getValueType(0);
1081 EVT NVT = EVT();
1082 SDLoc dl(N);
1083
1084 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1085 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1086 // match. Look for an appropriate libcall.
1087 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1088 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1089 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1090 NVT = (MVT::SimpleValueType)t;
1091 // The source needs to big enough to hold the operand.
1092 if (NVT.bitsGE(SVT))
1093 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
1094 }
1095 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1096
1097 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1098 // Sign/zero extend the argument if the libcall takes a larger type.
1100 NVT, N->getOperand(IsStrict ? 1 : 0));
1102 CallOptions.setIsSigned(Signed);
1103 CallOptions.setTypeListBeforeSoften(SVT, RVT);
1104 std::pair<SDValue, SDValue> Tmp =
1105 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
1106 Op, CallOptions, dl, Chain);
1107
1108 if (IsStrict)
1109 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1110 return Tmp.first;
1111}
1112
1113SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1114 // Expand and soften recursively.
1115 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
1116 return SDValue();
1117}
1118
1119SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1120 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
1121 return SDValue();
1122}
1123
1124//===----------------------------------------------------------------------===//
1125// Convert Float Operand to Integer
1126//===----------------------------------------------------------------------===//
1127
1128bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1129 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1130 SDValue Res = SDValue();
1131
1132 switch (N->getOpcode()) {
1133 default:
1134#ifndef NDEBUG
1135 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1136 N->dump(&DAG); dbgs() << "\n";
1137#endif
1138 report_fatal_error("Do not know how to soften this operator's operand!");
1139
1140 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1141 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1143 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1144 case ISD::FP_TO_BF16:
1147 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1150 case ISD::FP_TO_SINT:
1151 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1154 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1155 case ISD::STRICT_LROUND:
1156 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1158 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1159 case ISD::STRICT_LRINT:
1160 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1161 case ISD::STRICT_LLRINT:
1162 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1163 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1164 case ISD::STRICT_FSETCC:
1166 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1167 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1168 case ISD::ATOMIC_STORE:
1169 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1170 break;
1171 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1172 case ISD::FAKE_USE:
1173 Res = SoftenFloatOp_FAKE_USE(N);
1174 break;
1175 }
1176
1177 // If the result is null, the sub-method took care of registering results etc.
1178 if (!Res.getNode()) return false;
1179
1180 // If the result is N, the sub-method updated N in place. Tell the legalizer
1181 // core about this to re-analyze.
1182 if (Res.getNode() == N)
1183 return true;
1184
1185 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1186 "Invalid operand softening");
1187
1188 ReplaceValueWith(SDValue(N, 0), Res);
1189 return false;
1190}
1191
1192SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1193 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1194
1195 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1196}
1197
1198SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1199 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1200 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1201 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1202 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1203 N->getOpcode() == ISD::FP_TO_BF16 ||
1204 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1205 N->getOpcode() == ISD::STRICT_FP_ROUND);
1206
1207 bool IsStrict = N->isStrictFPOpcode();
1208 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1209 EVT SVT = Op.getValueType();
1210 EVT RVT = N->getValueType(0);
1211 EVT FloatRVT = RVT;
1212 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1213 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1214 FloatRVT = MVT::f16;
1215 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1216 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1217 FloatRVT = MVT::bf16;
1218
1219 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1220 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1221
1222 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1223 Op = GetSoftenedFloat(Op);
1225 CallOptions.setTypeListBeforeSoften(SVT, RVT);
1226 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1227 CallOptions, SDLoc(N),
1228 Chain);
1229 if (IsStrict) {
1230 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1231 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1232 return SDValue();
1233 }
1234 return Tmp.first;
1235}
1236
1237SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1238 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1239 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1240
1241 EVT VT = NewLHS.getValueType();
1242 NewLHS = GetSoftenedFloat(NewLHS);
1243 NewRHS = GetSoftenedFloat(NewRHS);
1244 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1245 N->getOperand(2), N->getOperand(3));
1246
1247 // If softenSetCCOperands returned a scalar, we need to compare the result
1248 // against zero to select between true and false values.
1249 if (!NewRHS.getNode()) {
1250 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1251 CCCode = ISD::SETNE;
1252 }
1253
1254 // Update N to have the operands specified.
1255 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1256 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1257 N->getOperand(4)),
1258 0);
1259}
1260
1261// Even if the result type is legal, no libcall may exactly match. (e.g. We
1262// don't have FP-i8 conversions) This helper method looks for an appropriate
1263// promoted libcall.
1264static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1265 bool Signed) {
1266 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1267 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1268 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1269 ++IntVT) {
1270 Promoted = (MVT::SimpleValueType)IntVT;
1271 // The type needs to big enough to hold the result.
1272 if (Promoted.bitsGE(RetVT))
1273 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1274 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1275 }
1276 return LC;
1277}
1278
1279SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1280 bool IsStrict = N->isStrictFPOpcode();
1281 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1282 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1283
1284 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1285 EVT SVT = Op.getValueType();
1286 EVT RVT = N->getValueType(0);
1287 EVT NVT = EVT();
1288 SDLoc dl(N);
1289
1290 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1291 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1292 // match, eg. we don't have fp -> i8 conversions.
1293 // Look for an appropriate libcall.
1294 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1295 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1296 "Unsupported FP_TO_XINT!");
1297
1298 Op = GetSoftenedFloat(Op);
1299 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1301 CallOptions.setTypeListBeforeSoften(SVT, RVT);
1302 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1303 CallOptions, dl, Chain);
1304
1305 // Truncate the result if the libcall returns a larger type.
1306 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1307
1308 if (!IsStrict)
1309 return Res;
1310
1311 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1312 ReplaceValueWith(SDValue(N, 0), Res);
1313 return SDValue();
1314}
1315
1316SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1317 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1318 return Res;
1319}
1320
1321SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1322 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1323 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1324
1325 EVT VT = NewLHS.getValueType();
1326 NewLHS = GetSoftenedFloat(NewLHS);
1327 NewRHS = GetSoftenedFloat(NewRHS);
1328 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1329 N->getOperand(0), N->getOperand(1));
1330
1331 // If softenSetCCOperands returned a scalar, we need to compare the result
1332 // against zero to select between true and false values.
1333 if (!NewRHS.getNode()) {
1334 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1335 CCCode = ISD::SETNE;
1336 }
1337
1338 // Update N to have the operands specified.
1339 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1340 N->getOperand(2), N->getOperand(3),
1341 DAG.getCondCode(CCCode)),
1342 0);
1343}
1344
1345SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1346 bool IsStrict = N->isStrictFPOpcode();
1347 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1348 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1349 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1350 ISD::CondCode CCCode =
1351 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1352
1353 EVT VT = Op0.getValueType();
1354 SDValue NewLHS = GetSoftenedFloat(Op0);
1355 SDValue NewRHS = GetSoftenedFloat(Op1);
1356 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1357 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1358
1359 // Update N to have the operands specified.
1360 if (NewRHS.getNode()) {
1361 if (IsStrict)
1362 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1363 NewRHS, DAG.getCondCode(CCCode));
1364 else
1365 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1366 DAG.getCondCode(CCCode)), 0);
1367 }
1368
1369 // Otherwise, softenSetCCOperands returned a scalar, use it.
1370 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1371 "Unexpected setcc expansion!");
1372
1373 if (IsStrict) {
1374 ReplaceValueWith(SDValue(N, 0), NewLHS);
1375 ReplaceValueWith(SDValue(N, 1), Chain);
1376 return SDValue();
1377 }
1378 return NewLHS;
1379}
1380
1381SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1382 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1383 assert(OpNo == 1 && "Can only soften the stored value!");
1384 StoreSDNode *ST = cast<StoreSDNode>(N);
1385 SDValue Val = ST->getValue();
1386 SDLoc dl(N);
1387
1388 if (ST->isTruncatingStore())
1389 // Do an FP_ROUND followed by a non-truncating store.
1390 Val = BitConvertToInteger(
1391 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1392 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1393 else
1394 Val = GetSoftenedFloat(Val);
1395
1396 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1397 ST->getMemOperand());
1398}
1399
1400SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1401 assert(OpNo == 1 && "Can only soften the stored value!");
1402 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1403 SDValue Val = ST->getVal();
1404 EVT VT = Val.getValueType();
1405 SDLoc dl(N);
1406
1407 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1408
1409 SDValue NewVal = GetSoftenedFloat(Val);
1410 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1411 ST->getBasePtr(), ST->getMemOperand());
1412}
1413
1414SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1415 SDValue LHS = N->getOperand(0);
1416 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1417 SDLoc dl(N);
1418
1419 EVT LVT = LHS.getValueType();
1420 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1421 EVT RVT = RHS.getValueType();
1422
1423 unsigned LSize = LVT.getSizeInBits();
1424 unsigned RSize = RVT.getSizeInBits();
1425
1426 // Shift right or sign-extend it if the two operands have different types.
1427 int SizeDiff = RSize - LSize;
1428 if (SizeDiff > 0) {
1429 RHS =
1430 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1431 DAG.getConstant(SizeDiff, dl,
1432 TLI.getShiftAmountTy(RHS.getValueType(),
1433 DAG.getDataLayout())));
1434 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1435 } else if (SizeDiff < 0) {
1436 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1437 RHS =
1438 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1439 DAG.getConstant(-SizeDiff, dl,
1440 TLI.getShiftAmountTy(RHS.getValueType(),
1441 DAG.getDataLayout())));
1442 }
1443
1444 RHS = DAG.getBitcast(LVT, RHS);
1445 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1446}
1447
1448SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1449 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1450 bool IsStrict = N->isStrictFPOpcode();
1451 unsigned Offset = IsStrict ? 1 : 0;
1452 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1453 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1455 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1456 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
1457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1458 CallOptions, SDLoc(N),
1459 Chain);
1460 if (IsStrict) {
1461 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1462 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1463 return SDValue();
1464 }
1465
1466 return Tmp.first;
1467}
1468
1469SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1470 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1471 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1472 RTLIB::LROUND_F32,
1473 RTLIB::LROUND_F64,
1474 RTLIB::LROUND_F80,
1475 RTLIB::LROUND_F128,
1476 RTLIB::LROUND_PPCF128));
1477}
1478
1479SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1480 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1481 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1482 RTLIB::LLROUND_F32,
1483 RTLIB::LLROUND_F64,
1484 RTLIB::LLROUND_F80,
1485 RTLIB::LLROUND_F128,
1486 RTLIB::LLROUND_PPCF128));
1487}
1488
1489SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1490 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1491 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1492 RTLIB::LRINT_F32,
1493 RTLIB::LRINT_F64,
1494 RTLIB::LRINT_F80,
1495 RTLIB::LRINT_F128,
1496 RTLIB::LRINT_PPCF128));
1497}
1498
1499SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1500 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1501 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1502 RTLIB::LLRINT_F32,
1503 RTLIB::LLRINT_F64,
1504 RTLIB::LLRINT_F80,
1505 RTLIB::LLRINT_F128,
1506 RTLIB::LLRINT_PPCF128));
1507}
1508
1509SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1510 SDValue Op1 = BitConvertToInteger(N->getOperand(1));
1511 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1512 N->getOperand(0), Op1);
1513}
1514
1515//===----------------------------------------------------------------------===//
1516// Float Result Expansion
1517//===----------------------------------------------------------------------===//
1518
1519/// ExpandFloatResult - This method is called when the specified result of the
1520/// specified node is found to need expansion. At this point, the node may also
1521/// have invalid operands or may have other results that need promotion, we just
1522/// know that (at least) one result needs expansion.
1523void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1524 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1525 SDValue Lo, Hi;
1526 Lo = Hi = SDValue();
1527
1528 // See if the target wants to custom expand this node.
1529 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1530 return;
1531
1532 switch (N->getOpcode()) {
1533 default:
1534#ifndef NDEBUG
1535 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1536 N->dump(&DAG); dbgs() << "\n";
1537#endif
1538 report_fatal_error("Do not know how to expand the result of this "
1539 "operator!");
1540 // clang-format off
1541 case ISD::POISON:
1542 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1543 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1544 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1545
1546 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1547 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1548 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1549 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1550 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1551 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1552
1553 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1554 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1555 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1557 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1559 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1560 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1561 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1562 case ISD::STRICT_FADD:
1563 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1564 case ISD::STRICT_FACOS:
1565 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1566 case ISD::STRICT_FASIN:
1567 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1568 case ISD::STRICT_FATAN:
1569 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1570 case ISD::STRICT_FATAN2:
1571 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1572 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1573 case ISD::STRICT_FCEIL:
1574 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1575 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1576 case ISD::STRICT_FCOS:
1577 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1578 case ISD::STRICT_FCOSH:
1579 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1580 case ISD::STRICT_FDIV:
1581 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1582 case ISD::STRICT_FEXP:
1583 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1584 case ISD::STRICT_FEXP2:
1585 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1586 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1587 case ISD::STRICT_FFLOOR:
1588 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1589 case ISD::STRICT_FLOG:
1590 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1591 case ISD::STRICT_FLOG2:
1592 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1593 case ISD::STRICT_FLOG10:
1594 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1595 case ISD::STRICT_FMA:
1596 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1597 case ISD::STRICT_FMUL:
1598 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1600 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1601 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1603 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1604 case ISD::STRICT_FPOW:
1605 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1606 case ISD::STRICT_FPOWI:
1607 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1608 case ISD::FLDEXP:
1609 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1610 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1611 case ISD::STRICT_FRINT:
1612 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1613 case ISD::STRICT_FROUND:
1614 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1616 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1617 case ISD::STRICT_FSIN:
1618 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1619 case ISD::STRICT_FSINH:
1620 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1621 case ISD::STRICT_FSQRT:
1622 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1623 case ISD::STRICT_FSUB:
1624 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1625 case ISD::STRICT_FTAN:
1626 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1627 case ISD::STRICT_FTANH:
1628 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1629 case ISD::STRICT_FTRUNC:
1630 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1631 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1634 case ISD::SINT_TO_FP:
1635 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1636 case ISD::STRICT_FREM:
1637 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1638 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1639 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1640 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1641 // clang-format on
1642 }
1643
1644 // If Lo/Hi is null, the sub-method took care of registering results etc.
1645 if (Lo.getNode())
1646 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1647}
1648
1649void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1650 SDValue &Hi) {
1651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1652 assert(NVT.getSizeInBits() == 64 &&
1653 "Do not know how to expand this float constant!");
1654 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1655 SDLoc dl(N);
1656 const fltSemantics &Sem = NVT.getFltSemantics();
1657 Lo = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 64)), dl, NVT);
1658 Hi = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 0)), dl, NVT);
1659}
1660
1661void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1662 SDValue &Lo, SDValue &Hi) {
1663 bool IsStrict = N->isStrictFPOpcode();
1664 unsigned Offset = IsStrict ? 1 : 0;
1665 SDValue Op = N->getOperand(0 + Offset);
1666 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1668 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1669 Op, CallOptions, SDLoc(N),
1670 Chain);
1671 if (IsStrict)
1672 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1673 GetPairElements(Tmp.first, Lo, Hi);
1674}
1675
1676void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1677 SDValue &Lo, SDValue &Hi) {
1678 bool IsStrict = N->isStrictFPOpcode();
1679 unsigned Offset = IsStrict ? 1 : 0;
1680 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1681 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1683 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1684 Ops, CallOptions, SDLoc(N),
1685 Chain);
1686 if (IsStrict)
1687 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1688 GetPairElements(Tmp.first, Lo, Hi);
1689}
1690
1691void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1692 ExpandFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(N->getValueType(0)),
1693 /*CallRetResNo=*/0);
1694}
1695
1696void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1697 ExpandFloatRes_UnaryWithTwoFPResults(N, RTLIB::getSINCOS(N->getValueType(0)));
1698}
1699
1700void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1701 ExpandFloatRes_UnaryWithTwoFPResults(N,
1702 RTLIB::getSINCOSPI(N->getValueType(0)));
1703}
1704
1705void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1706 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1707 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1709 DAG.expandMultipleResultFPLibCall(LC, N, Results, CallRetResNo);
1710 for (auto [ResNo, Res] : enumerate(Results)) {
1711 SDValue Lo, Hi;
1712 GetPairElements(Res, Lo, Hi);
1713 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1714 }
1715}
1716
1717void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1718 SDValue &Hi) {
1719 assert(N->getValueType(0) == MVT::ppcf128 &&
1720 "Logic only correct for ppcf128!");
1721 SDLoc dl(N);
1722 SDValue Tmp;
1723 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1724 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1725 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1726 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1727 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1728 ISD::SETEQ);
1729}
1730
1731void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1732 SDValue &Hi) {
1733 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1734 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1735 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1736 RTLIB::FMIN_PPCF128), Lo, Hi);
1737}
1738
1739void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1740 SDValue &Hi) {
1741 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1742 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1743 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1744 RTLIB::FMAX_PPCF128), Lo, Hi);
1745}
1746
1747void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1748 SDValue &Hi) {
1749 ExpandFloatRes_Binary(
1750 N,
1751 GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
1752 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
1753 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128),
1754 Lo, Hi);
1755}
1756
1757void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1758 SDValue &Hi) {
1759 ExpandFloatRes_Binary(
1760 N,
1761 GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
1762 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
1763 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128),
1764 Lo, Hi);
1765}
1766
1767void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1768 SDValue &Hi) {
1769 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1770 RTLIB::ADD_F32, RTLIB::ADD_F64,
1771 RTLIB::ADD_F80, RTLIB::ADD_F128,
1772 RTLIB::ADD_PPCF128), Lo, Hi);
1773}
1774
1775void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1776 SDValue &Hi) {
1777 ExpandFloatRes_Unary(N,
1778 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32,
1779 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1780 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1781 Lo, Hi);
1782}
1783
1784void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1785 SDValue &Hi) {
1786 ExpandFloatRes_Unary(N,
1787 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32,
1788 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1789 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1790 Lo, Hi);
1791}
1792
1793void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1794 SDValue &Hi) {
1795 ExpandFloatRes_Unary(N,
1796 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32,
1797 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1798 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1799 Lo, Hi);
1800}
1801
1802void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1803 SDValue &Hi) {
1804 ExpandFloatRes_Binary(N,
1805 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32,
1806 RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,
1807 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128),
1808 Lo, Hi);
1809}
1810
1811void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1814 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1815 RTLIB::CBRT_F128,
1816 RTLIB::CBRT_PPCF128), Lo, Hi);
1817}
1818
1819void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1820 SDValue &Lo, SDValue &Hi) {
1821 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1822 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1823 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1824 RTLIB::CEIL_PPCF128), Lo, Hi);
1825}
1826
1827void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1828 SDValue &Lo, SDValue &Hi) {
1829 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1830 RTLIB::COPYSIGN_F32,
1831 RTLIB::COPYSIGN_F64,
1832 RTLIB::COPYSIGN_F80,
1833 RTLIB::COPYSIGN_F128,
1834 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1835}
1836
1837void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1838 SDValue &Lo, SDValue &Hi) {
1839 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1840 RTLIB::COS_F32, RTLIB::COS_F64,
1841 RTLIB::COS_F80, RTLIB::COS_F128,
1842 RTLIB::COS_PPCF128), Lo, Hi);
1843}
1844
1845void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1846 SDValue &Hi) {
1847 ExpandFloatRes_Unary(N,
1848 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32,
1849 RTLIB::COSH_F64, RTLIB::COSH_F80,
1850 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1851 Lo, Hi);
1852}
1853
1854void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1855 SDValue &Hi) {
1856 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1857 RTLIB::DIV_F32,
1858 RTLIB::DIV_F64,
1859 RTLIB::DIV_F80,
1860 RTLIB::DIV_F128,
1861 RTLIB::DIV_PPCF128), Lo, Hi);
1862}
1863
1864void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1865 SDValue &Lo, SDValue &Hi) {
1866 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1867 RTLIB::EXP_F32, RTLIB::EXP_F64,
1868 RTLIB::EXP_F80, RTLIB::EXP_F128,
1869 RTLIB::EXP_PPCF128), Lo, Hi);
1870}
1871
1872void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1873 SDValue &Lo, SDValue &Hi) {
1874 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1875 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1876 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1877 RTLIB::EXP2_PPCF128), Lo, Hi);
1878}
1879
1880void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1881 SDValue &Hi) {
1882 ExpandFloatRes_Unary(N,
1883 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1884 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1885 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1886 Lo, Hi);
1887}
1888
1889void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1890 SDValue &Lo, SDValue &Hi) {
1891 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1892 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1893 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1894 RTLIB::FLOOR_PPCF128), Lo, Hi);
1895}
1896
1897void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1898 SDValue &Lo, SDValue &Hi) {
1899 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1900 RTLIB::LOG_F32, RTLIB::LOG_F64,
1901 RTLIB::LOG_F80, RTLIB::LOG_F128,
1902 RTLIB::LOG_PPCF128), Lo, Hi);
1903}
1904
1905void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1906 SDValue &Lo, SDValue &Hi) {
1907 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1908 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1909 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1910 RTLIB::LOG2_PPCF128), Lo, Hi);
1911}
1912
1913void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1914 SDValue &Lo, SDValue &Hi) {
1915 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1916 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1917 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1918 RTLIB::LOG10_PPCF128), Lo, Hi);
1919}
1920
1921void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1922 SDValue &Hi) {
1923 bool IsStrict = N->isStrictFPOpcode();
1924 unsigned Offset = IsStrict ? 1 : 0;
1925 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1926 N->getOperand(2 + Offset) };
1927 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1929 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1930 RTLIB::FMA_F32,
1931 RTLIB::FMA_F64,
1932 RTLIB::FMA_F80,
1933 RTLIB::FMA_F128,
1934 RTLIB::FMA_PPCF128),
1935 N->getValueType(0), Ops, CallOptions,
1936 SDLoc(N), Chain);
1937 if (IsStrict)
1938 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1939 GetPairElements(Tmp.first, Lo, Hi);
1940}
1941
1942void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1943 SDValue &Hi) {
1944 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1945 RTLIB::MUL_F32,
1946 RTLIB::MUL_F64,
1947 RTLIB::MUL_F80,
1948 RTLIB::MUL_F128,
1949 RTLIB::MUL_PPCF128), Lo, Hi);
1950}
1951
1952void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1953 SDValue &Lo, SDValue &Hi) {
1954 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1955 RTLIB::NEARBYINT_F32,
1956 RTLIB::NEARBYINT_F64,
1957 RTLIB::NEARBYINT_F80,
1958 RTLIB::NEARBYINT_F128,
1959 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1960}
1961
1962void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1963 SDValue &Hi) {
1964 SDLoc dl(N);
1965 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1966 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1967 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1968}
1969
1970void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1971 SDValue &Hi) {
1972 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1973 SDLoc dl(N);
1974 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1975}
1976
1977void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1978 SDValue &Hi) {
1979 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1980 SDLoc dl(N);
1981 bool IsStrict = N->isStrictFPOpcode();
1982
1983 SDValue Chain;
1984 if (IsStrict) {
1985 // If the expanded type is the same as the input type, just bypass the node.
1986 if (NVT == N->getOperand(1).getValueType()) {
1987 Hi = N->getOperand(1);
1988 Chain = N->getOperand(0);
1989 } else {
1990 // Other we need to extend.
1991 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1992 { N->getOperand(0), N->getOperand(1) });
1993 Chain = Hi.getValue(1);
1994 }
1995 } else {
1996 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1997 }
1998
1999 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2000
2001 if (IsStrict)
2002 ReplaceValueWith(SDValue(N, 1), Chain);
2003}
2004
2005void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
2006 SDValue &Lo, SDValue &Hi) {
2007 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2008 RTLIB::POW_F32, RTLIB::POW_F64,
2009 RTLIB::POW_F80, RTLIB::POW_F128,
2010 RTLIB::POW_PPCF128), Lo, Hi);
2011}
2012
2013void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
2014 SDValue &Lo, SDValue &Hi) {
2015 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
2016}
2017
2018void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
2019 SDValue &Hi) {
2020 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
2021}
2022
2023void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
2024 SDValue &Lo, SDValue &Hi) {
2025 assert(N->getValueType(0) == MVT::ppcf128 &&
2026 "Logic only correct for ppcf128!");
2027
2028 SDLoc dl(N);
2029 GetExpandedFloat(N->getOperand(0), Lo, Hi);
2030 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
2031 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
2032}
2033
2034void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
2035 SDValue &Lo, SDValue &Hi) {
2036 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2037 RTLIB::REM_F32, RTLIB::REM_F64,
2038 RTLIB::REM_F80, RTLIB::REM_F128,
2039 RTLIB::REM_PPCF128), Lo, Hi);
2040}
2041
2042void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
2043 SDValue &Lo, SDValue &Hi) {
2044 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2045 RTLIB::RINT_F32, RTLIB::RINT_F64,
2046 RTLIB::RINT_F80, RTLIB::RINT_F128,
2047 RTLIB::RINT_PPCF128), Lo, Hi);
2048}
2049
2050void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
2051 SDValue &Lo, SDValue &Hi) {
2052 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2053 RTLIB::ROUND_F32,
2054 RTLIB::ROUND_F64,
2055 RTLIB::ROUND_F80,
2056 RTLIB::ROUND_F128,
2057 RTLIB::ROUND_PPCF128), Lo, Hi);
2058}
2059
2060void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
2061 SDValue &Lo, SDValue &Hi) {
2062 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2063 RTLIB::ROUNDEVEN_F32,
2064 RTLIB::ROUNDEVEN_F64,
2065 RTLIB::ROUNDEVEN_F80,
2066 RTLIB::ROUNDEVEN_F128,
2067 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
2068}
2069
2070void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
2071 SDValue &Lo, SDValue &Hi) {
2072 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2073 RTLIB::SIN_F32, RTLIB::SIN_F64,
2074 RTLIB::SIN_F80, RTLIB::SIN_F128,
2075 RTLIB::SIN_PPCF128), Lo, Hi);
2076}
2077
2078void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
2079 SDValue &Hi) {
2080 ExpandFloatRes_Unary(N,
2081 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32,
2082 RTLIB::SINH_F64, RTLIB::SINH_F80,
2083 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
2084 Lo, Hi);
2085}
2086
2087void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
2088 SDValue &Lo, SDValue &Hi) {
2089 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2090 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
2091 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
2092 RTLIB::SQRT_PPCF128), Lo, Hi);
2093}
2094
2095void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2096 SDValue &Hi) {
2097 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2098 RTLIB::SUB_F32,
2099 RTLIB::SUB_F64,
2100 RTLIB::SUB_F80,
2101 RTLIB::SUB_F128,
2102 RTLIB::SUB_PPCF128), Lo, Hi);
2103}
2104
2105void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2106 SDValue &Hi) {
2107 ExpandFloatRes_Unary(N,
2108 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
2109 RTLIB::TAN_F64, RTLIB::TAN_F80,
2110 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
2111 Lo, Hi);
2112}
2113
2114void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2115 SDValue &Hi) {
2116 ExpandFloatRes_Unary(N,
2117 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32,
2118 RTLIB::TANH_F64, RTLIB::TANH_F80,
2119 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
2120 Lo, Hi);
2121}
2122
2123void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2124 SDValue &Lo, SDValue &Hi) {
2125 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2126 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
2127 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
2128 RTLIB::TRUNC_PPCF128), Lo, Hi);
2129}
2130
2131void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2132 SDValue &Hi) {
2133 if (ISD::isNormalLoad(N)) {
2134 ExpandRes_NormalLoad(N, Lo, Hi);
2135 return;
2136 }
2137
2138 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2139 LoadSDNode *LD = cast<LoadSDNode>(N);
2140 SDValue Chain = LD->getChain();
2141 SDValue Ptr = LD->getBasePtr();
2142 SDLoc dl(N);
2143
2144 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
2145 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2146 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2147
2148 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
2149 LD->getMemoryVT(), LD->getMemOperand());
2150
2151 // Remember the chain.
2152 Chain = Hi.getValue(1);
2153
2154 // The low part is zero.
2155 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2156
2157 // Modified the chain - switch anything that used the old chain to use the
2158 // new one.
2159 ReplaceValueWith(SDValue(LD, 1), Chain);
2160}
2161
2162void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2163 SDValue &Hi) {
2164 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2165 EVT VT = N->getValueType(0);
2166 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2167 bool Strict = N->isStrictFPOpcode();
2168 SDValue Src = N->getOperand(Strict ? 1 : 0);
2169 EVT SrcVT = Src.getValueType();
2170 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2171 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2172 SDLoc dl(N);
2173 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
2174
2175 // TODO: Any other flags to propagate?
2177 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2178
2179 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2180 // When promoting partial word types to i32 we must honor the signedness,
2181 // though.
2182 if (SrcVT.bitsLE(MVT::i32)) {
2183 // The integer can be represented exactly in an f64.
2184 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2185 if (Strict) {
2186 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
2187 {Chain, Src}, Flags);
2188 Chain = Hi.getValue(1);
2189 } else
2190 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
2191 } else {
2192 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2193 if (SrcVT.bitsLE(MVT::i64)) {
2195 MVT::i64, Src);
2196 LC = RTLIB::SINTTOFP_I64_PPCF128;
2197 } else if (SrcVT.bitsLE(MVT::i128)) {
2198 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
2199 LC = RTLIB::SINTTOFP_I128_PPCF128;
2200 }
2201 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2202
2204 CallOptions.setIsSigned(true);
2205 std::pair<SDValue, SDValue> Tmp =
2206 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
2207 if (Strict)
2208 Chain = Tmp.second;
2209 GetPairElements(Tmp.first, Lo, Hi);
2210 }
2211
2212 // No need to complement for unsigned 32-bit integers
2213 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
2214 if (Strict)
2215 ReplaceValueWith(SDValue(N, 1), Chain);
2216
2217 return;
2218 }
2219
2220 // Unsigned - fix up the SINT_TO_FP value just calculated.
2221 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2222 // keep semantics correctness if the integer is not exactly representable
2223 // here. See ExpandLegalINT_TO_FP.
2224 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
2225 SrcVT = Src.getValueType();
2226
2227 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2228 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2229 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2230 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2231 ArrayRef<uint64_t> Parts;
2232
2233 switch (SrcVT.getSimpleVT().SimpleTy) {
2234 default:
2235 llvm_unreachable("Unsupported UINT_TO_FP!");
2236 case MVT::i32:
2237 Parts = TwoE32;
2238 break;
2239 case MVT::i64:
2240 Parts = TwoE64;
2241 break;
2242 case MVT::i128:
2243 Parts = TwoE128;
2244 break;
2245 }
2246
2247 // TODO: Are there other fast-math-flags to propagate to this FADD?
2248 SDValue NewLo = DAG.getConstantFP(
2249 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
2250 if (Strict) {
2251 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
2252 {Chain, Hi, NewLo}, Flags);
2253 Chain = Lo.getValue(1);
2254 ReplaceValueWith(SDValue(N, 1), Chain);
2255 } else
2256 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
2257 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
2258 Lo, Hi, ISD::SETLT);
2259 GetPairElements(Lo, Lo, Hi);
2260}
2261
2262
2263//===----------------------------------------------------------------------===//
2264// Float Operand Expansion
2265//===----------------------------------------------------------------------===//
2266
2267/// ExpandFloatOperand - This method is called when the specified operand of the
2268/// specified node is found to need expansion. At this point, all of the result
2269/// types of the node are known to be legal, but other operands of the node may
2270/// need promotion or expansion as well as the specified one.
2271bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2272 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2273 SDValue Res = SDValue();
2274
2275 // See if the target wants to custom expand this node.
2276 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2277 return false;
2278
2279 switch (N->getOpcode()) {
2280 default:
2281#ifndef NDEBUG
2282 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2283 N->dump(&DAG); dbgs() << "\n";
2284#endif
2285 report_fatal_error("Do not know how to expand this operator's operand!");
2286
2287 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2288 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2289 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2290
2291 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2292 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2294 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2297 case ISD::FP_TO_SINT:
2298 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2299 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2300 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2301 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2302 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2303 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2304 case ISD::STRICT_FSETCC:
2306 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2307 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
2308 OpNo); break;
2309 }
2310
2311 // If the result is null, the sub-method took care of registering results etc.
2312 if (!Res.getNode()) return false;
2313
2314 // If the result is N, the sub-method updated N in place. Tell the legalizer
2315 // core about this.
2316 if (Res.getNode() == N)
2317 return true;
2318
2319 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2320 "Invalid operand expansion");
2321
2322 ReplaceValueWith(SDValue(N, 0), Res);
2323 return false;
2324}
2325
2326/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2327/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2328void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2329 SDValue &NewRHS,
2330 ISD::CondCode &CCCode,
2331 const SDLoc &dl, SDValue &Chain,
2332 bool IsSignaling) {
2333 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2334 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2335 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2336
2337 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2338
2339 // FIXME: This generated code sucks. We want to generate
2340 // FCMPU crN, hi1, hi2
2341 // BNE crN, L:
2342 // FCMPU crN, lo1, lo2
2343 // The following can be improved, but not that much.
2344 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2345 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2346 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2347 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2348 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2349 RHSLo, CCCode, OutputChain, IsSignaling);
2350 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2351 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2352 Tmp1 =
2353 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2354 ISD::SETUNE, OutputChain, IsSignaling);
2355 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2356 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2357 RHSHi, CCCode, OutputChain, IsSignaling);
2358 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2359 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2360 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2361 NewRHS = SDValue(); // LHS is the result, not a compare.
2362 Chain = OutputChain;
2363}
2364
2365SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2366 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2367 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2368 SDValue Chain;
2369 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2370
2371 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2372 // against zero to select between true and false values.
2373 if (!NewRHS.getNode()) {
2374 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2375 CCCode = ISD::SETNE;
2376 }
2377
2378 // Update N to have the operands specified.
2379 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2380 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2381 N->getOperand(4)), 0);
2382}
2383
2384SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2385 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2386 "Logic only correct for ppcf128!");
2387 SDValue Lo, Hi;
2388 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2389 // The ppcf128 value is providing only the sign; take it from the
2390 // higher-order double (which must have the larger magnitude).
2391 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2392 N->getValueType(0), N->getOperand(0), Hi);
2393}
2394
2395SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2396 bool IsStrict = N->isStrictFPOpcode();
2397 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2398 "Logic only correct for ppcf128!");
2399 SDValue Lo, Hi;
2400 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2401
2402 if (!IsStrict)
2403 // Round it the rest of the way (e.g. to f32) if needed.
2404 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2405 N->getValueType(0), Hi, N->getOperand(1));
2406
2407 // Eliminate the node if the input float type is the same as the output float
2408 // type.
2409 if (Hi.getValueType() == N->getValueType(0)) {
2410 // Connect the output chain to the input chain, unlinking the node.
2411 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2412 ReplaceValueWith(SDValue(N, 0), Hi);
2413 return SDValue();
2414 }
2415
2417 {N->getValueType(0), MVT::Other},
2418 {N->getOperand(0), Hi, N->getOperand(2)});
2419 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2420 ReplaceValueWith(SDValue(N, 0), Expansion);
2421 return SDValue();
2422}
2423
2424SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2425 EVT RVT = N->getValueType(0);
2426 SDLoc dl(N);
2427
2428 bool IsStrict = N->isStrictFPOpcode();
2429 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2430 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2431 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2432 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2433
2434 EVT NVT;
2435 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2436 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2437 "Unsupported FP_TO_XINT!");
2439 std::pair<SDValue, SDValue> Tmp =
2440 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2441 if (!IsStrict)
2442 return Tmp.first;
2443
2444 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2445 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2446 return SDValue();
2447}
2448
2449SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2450 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2451 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2452 SDValue Chain;
2453 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2454
2455 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2456 // against zero to select between true and false values.
2457 if (!NewRHS.getNode()) {
2458 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2459 CCCode = ISD::SETNE;
2460 }
2461
2462 // Update N to have the operands specified.
2463 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2464 N->getOperand(2), N->getOperand(3),
2465 DAG.getCondCode(CCCode)), 0);
2466}
2467
2468SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2469 bool IsStrict = N->isStrictFPOpcode();
2470 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2471 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2472 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2473 ISD::CondCode CCCode =
2474 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2475 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2476 N->getOpcode() == ISD::STRICT_FSETCCS);
2477
2478 // FloatExpandSetCCOperands always returned a scalar.
2479 assert(!NewRHS.getNode() && "Expect to return scalar");
2480 assert(NewLHS.getValueType() == N->getValueType(0) &&
2481 "Unexpected setcc expansion!");
2482 if (Chain) {
2483 ReplaceValueWith(SDValue(N, 0), NewLHS);
2484 ReplaceValueWith(SDValue(N, 1), Chain);
2485 return SDValue();
2486 }
2487 return NewLHS;
2488}
2489
2490SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2491 if (ISD::isNormalStore(N))
2492 return ExpandOp_NormalStore(N, OpNo);
2493
2494 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2495 assert(OpNo == 1 && "Can only expand the stored value so far");
2496 StoreSDNode *ST = cast<StoreSDNode>(N);
2497
2498 SDValue Chain = ST->getChain();
2499 SDValue Ptr = ST->getBasePtr();
2500
2501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2502 ST->getValue().getValueType());
2503 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2504 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2505 (void)NVT;
2506
2507 SDValue Lo, Hi;
2508 GetExpandedOp(ST->getValue(), Lo, Hi);
2509
2510 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2511 ST->getMemoryVT(), ST->getMemOperand());
2512}
2513
2514SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2515 EVT RVT = N->getValueType(0);
2516 EVT RetVT = N->getOperand(0).getValueType();
2518 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2519 RTLIB::LROUND_F32,
2520 RTLIB::LROUND_F64,
2521 RTLIB::LROUND_F80,
2522 RTLIB::LROUND_F128,
2523 RTLIB::LROUND_PPCF128),
2524 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2525}
2526
2527SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2528 EVT RVT = N->getValueType(0);
2529 EVT RetVT = N->getOperand(0).getValueType();
2531 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2532 RTLIB::LLROUND_F32,
2533 RTLIB::LLROUND_F64,
2534 RTLIB::LLROUND_F80,
2535 RTLIB::LLROUND_F128,
2536 RTLIB::LLROUND_PPCF128),
2537 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2538}
2539
2540SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2541 EVT RVT = N->getValueType(0);
2542 EVT RetVT = N->getOperand(0).getValueType();
2544 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2545 RTLIB::LRINT_F32,
2546 RTLIB::LRINT_F64,
2547 RTLIB::LRINT_F80,
2548 RTLIB::LRINT_F128,
2549 RTLIB::LRINT_PPCF128),
2550 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2551}
2552
2553SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2554 EVT RVT = N->getValueType(0);
2555 EVT RetVT = N->getOperand(0).getValueType();
2557 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2558 RTLIB::LLRINT_F32,
2559 RTLIB::LLRINT_F64,
2560 RTLIB::LLRINT_F80,
2561 RTLIB::LLRINT_F128,
2562 RTLIB::LLRINT_PPCF128),
2563 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2564}
2565
2566//===----------------------------------------------------------------------===//
2567// Float Operand Promotion
2568//===----------------------------------------------------------------------===//
2569//
2570
2572 if (OpVT == MVT::f16)
2573 return ISD::FP16_TO_FP;
2574 if (RetVT == MVT::f16)
2575 return ISD::FP_TO_FP16;
2576 if (OpVT == MVT::bf16)
2577 return ISD::BF16_TO_FP;
2578 if (RetVT == MVT::bf16)
2579 return ISD::FP_TO_BF16;
2580 report_fatal_error("Attempt at an invalid promotion-related conversion");
2581}
2582
2584 if (OpVT == MVT::f16)
2586 if (RetVT == MVT::f16)
2588 if (OpVT == MVT::bf16)
2590 if (RetVT == MVT::bf16)
2592 report_fatal_error("Attempt at an invalid promotion-related conversion");
2593}
2594
2595bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2596 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2597 SDValue R = SDValue();
2598
2599 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2600 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2601 return false;
2602 }
2603
2604 // Nodes that use a promotion-requiring floating point operand, but doesn't
2605 // produce a promotion-requiring floating point result, need to be legalized
2606 // to use the promoted float operand. Nodes that produce at least one
2607 // promotion-requiring floating point result have their operands legalized as
2608 // a part of PromoteFloatResult.
2609 // clang-format off
2610 switch (N->getOpcode()) {
2611 default:
2612 #ifndef NDEBUG
2613 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2614 N->dump(&DAG); dbgs() << "\n";
2615 #endif
2616 report_fatal_error("Do not know how to promote this operator's operand!");
2617
2618 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2619 case ISD::FAKE_USE:
2620 R = PromoteFloatOp_FAKE_USE(N, OpNo);
2621 break;
2622 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2623 case ISD::FP_TO_SINT:
2624 case ISD::FP_TO_UINT:
2625 case ISD::LROUND:
2626 case ISD::LLROUND:
2627 case ISD::LRINT:
2628 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2629 case ISD::AssertNoFPClass: R = PromoteFloatOp_AssertNoFPClass(N, OpNo); break;
2632 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2633 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2635 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2636 break;
2637 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2638 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2639 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2640 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2641 }
2642 // clang-format on
2643
2644 if (R.getNode())
2645 ReplaceValueWith(SDValue(N, 0), R);
2646 return false;
2647}
2648
2649SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2650 SDValue Op = N->getOperand(0);
2651 EVT OpVT = Op->getValueType(0);
2652
2653 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2654 EVT PromotedVT = Promoted->getValueType(0);
2655
2656 // Convert the promoted float value to the desired IVT.
2657 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2658 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2659 IVT, Promoted);
2660 // The final result type might not be an scalar so we need a bitcast. The
2661 // bitcast will be further legalized if needed.
2662 return DAG.getBitcast(N->getValueType(0), Convert);
2663}
2664
2665SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo) {
2666 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2667 SDValue Op = GetPromotedFloat(N->getOperand(OpNo));
2668 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0),
2669 Op);
2670}
2671
2672// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2673// PromoteFloatRes_FCOPYSIGN.
2674SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2675 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2676 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2677
2678 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2679 N->getOperand(0), Op1);
2680}
2681
2682// Convert the promoted float value to the desired integer type
2683SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2684 SDValue Op = GetPromotedFloat(N->getOperand(0));
2685 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2686}
2687
2688// Convert the promoted float value to the desired integer type
2689SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(SDNode *N,
2690 unsigned OpNo) {
2691 return GetPromotedFloat(N->getOperand(0));
2692}
2693
2694SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2695 unsigned OpNo) {
2696 SDValue Op = GetPromotedFloat(N->getOperand(0));
2697 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2698 N->getOperand(1));
2699}
2700
2701SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2702 SDValue Op = GetPromotedFloat(N->getOperand(0));
2703 EVT VT = N->getValueType(0);
2704
2705 // Desired VT is same as promoted type. Use promoted float directly.
2706 if (VT == Op->getValueType(0))
2707 return Op;
2708
2709 // Else, extend the promoted float value to the desired VT.
2710 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2711}
2712
2713SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2714 unsigned OpNo) {
2715 assert(OpNo == 1 && "Promoting unpromotable operand");
2716
2717 SDValue Op = GetPromotedFloat(N->getOperand(1));
2718 EVT VT = N->getValueType(0);
2719
2720 // Desired VT is same as promoted type. Use promoted float directly.
2721 if (VT == Op->getValueType(0)) {
2722 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2723 return Op;
2724 }
2725
2726 // Else, extend the promoted float value to the desired VT.
2727 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2728 N->getOperand(0), Op);
2729 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2730 return Res;
2731}
2732
2733// Promote the float operands used for comparison. The true- and false-
2734// operands have the same type as the result and are promoted, if needed, by
2735// PromoteFloatRes_SELECT_CC
2736SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2737 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2738 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2739
2740 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2741 LHS, RHS, N->getOperand(2), N->getOperand(3),
2742 N->getOperand(4));
2743}
2744
2745// Construct a SETCC that compares the promoted values and sets the conditional
2746// code.
2747SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2748 EVT VT = N->getValueType(0);
2749 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2750 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2751 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2752
2753 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2754
2755}
2756
2757// Lower the promoted Float down to the integer value of same size and construct
2758// a STORE of the integer value.
2759SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2760 StoreSDNode *ST = cast<StoreSDNode>(N);
2761 SDValue Val = ST->getValue();
2762 SDLoc DL(N);
2763
2764 SDValue Promoted = GetPromotedFloat(Val);
2765 EVT VT = ST->getOperand(1).getValueType();
2766 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2767
2768 SDValue NewVal;
2769 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2770 IVT, Promoted);
2771
2772 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2773 ST->getMemOperand());
2774}
2775
2776SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2777 unsigned OpNo) {
2778 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2779 SDValue Val = ST->getVal();
2780 SDLoc DL(N);
2781
2782 SDValue Promoted = GetPromotedFloat(Val);
2783 EVT VT = ST->getOperand(1).getValueType();
2784 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2785
2786 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2787 DL, IVT, Promoted);
2788
2789 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2790 ST->getBasePtr(), ST->getMemOperand());
2791}
2792
2793//===----------------------------------------------------------------------===//
2794// Float Result Promotion
2795//===----------------------------------------------------------------------===//
2796
2797void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2798 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2799 SDValue R = SDValue();
2800
2801 // See if the target wants to custom expand this node.
2802 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2803 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2804 return;
2805 }
2806
2807 switch (N->getOpcode()) {
2808 // These opcodes cannot appear if promotion of FP16 is done in the backend
2809 // instead of Clang
2810 case ISD::FP16_TO_FP:
2811 case ISD::FP_TO_FP16:
2812 default:
2813#ifndef NDEBUG
2814 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2815 N->dump(&DAG); dbgs() << "\n";
2816#endif
2817 report_fatal_error("Do not know how to promote this operator's result!");
2818
2819 case ISD::BITCAST:
2820 R = PromoteFloatRes_BITCAST(N);
2821 break;
2822 case ISD::FREEZE:
2823 R = PromoteFloatRes_FREEZE(N);
2824 break;
2825 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2827 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2828 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2829
2830 // Unary FP Operations
2831 case ISD::FABS:
2832 case ISD::FACOS:
2833 case ISD::FASIN:
2834 case ISD::FATAN:
2835 case ISD::FCBRT:
2836 case ISD::FCEIL:
2837 case ISD::FCOS:
2838 case ISD::FCOSH:
2839 case ISD::FEXP:
2840 case ISD::FEXP2:
2841 case ISD::FEXP10:
2842 case ISD::FFLOOR:
2843 case ISD::FLOG:
2844 case ISD::FLOG2:
2845 case ISD::FLOG10:
2846 case ISD::FNEARBYINT:
2847 case ISD::FNEG:
2848 case ISD::FRINT:
2849 case ISD::FROUND:
2850 case ISD::FROUNDEVEN:
2851 case ISD::FSIN:
2852 case ISD::FSINH:
2853 case ISD::FSQRT:
2854 case ISD::FTRUNC:
2855 case ISD::FTAN:
2856 case ISD::FTANH:
2857 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2859 R = PromoteFloatRes_AssertNoFPClass(N);
2860 break;
2861
2862 // Binary FP Operations
2863 case ISD::FADD:
2864 case ISD::FDIV:
2865 case ISD::FMAXIMUM:
2866 case ISD::FMINIMUM:
2867 case ISD::FMAXIMUMNUM:
2868 case ISD::FMINIMUMNUM:
2869 case ISD::FMAXNUM:
2870 case ISD::FMINNUM:
2871 case ISD::FMAXNUM_IEEE:
2872 case ISD::FMINNUM_IEEE:
2873 case ISD::FMUL:
2874 case ISD::FPOW:
2875 case ISD::FATAN2:
2876 case ISD::FREM:
2877 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2878
2879 case ISD::FMA: // FMA is same as FMAD
2880 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2881
2882 case ISD::FPOWI:
2883 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2884 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2885
2886 case ISD::FMODF:
2887 case ISD::FSINCOS:
2888 case ISD::FSINCOSPI:
2889 R = PromoteFloatRes_UnaryWithTwoFPResults(N);
2890 break;
2891 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2893 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2894 break;
2895 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2896 case ISD::ATOMIC_LOAD:
2897 R = PromoteFloatRes_ATOMIC_LOAD(N);
2898 break;
2899 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2900 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2901
2902 case ISD::SINT_TO_FP:
2903 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2904 case ISD::POISON:
2905 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2906 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2913 R = PromoteFloatRes_VECREDUCE(N);
2914 break;
2917 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2918 break;
2919 }
2920
2921 if (R.getNode())
2922 SetPromotedFloat(SDValue(N, ResNo), R);
2923}
2924
2925// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2926// At this point, it is not possible to determine if the bitcast value is
2927// eventually stored to memory or promoted to f32 or promoted to a floating
2928// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2929// STORE promotion handlers.
2930SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2931 EVT VT = N->getValueType(0);
2932 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2933 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2934 // bitcast will be legalized further if necessary.
2935 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2936 N->getOperand(0).getValueType().getSizeInBits());
2937 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2938 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2939}
2940
2941SDValue DAGTypeLegalizer::PromoteFloatRes_FREEZE(SDNode *N) {
2942 EVT VT = N->getValueType(0);
2943 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2944 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2945 // bitcast will be legalized further if necessary.
2946 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2947 N->getOperand(0).getValueType().getSizeInBits());
2948 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2949 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT,
2950 DAG.getFreeze(Cast));
2951}
2952
2953SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2954 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2955 EVT VT = N->getValueType(0);
2956 SDLoc DL(N);
2957
2958 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2959 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2960 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2961 IVT);
2962
2963 // Convert the Constant to the desired FP type
2964 // FIXME We might be able to do the conversion during compilation and get rid
2965 // of it from the object code
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2967 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2968}
2969
2970// If the Index operand is a constant, try to redirect the extract operation to
2971// the correct legalized vector. If not, bit-convert the input vector to
2972// equivalent integer vector. Extract the element as an (bit-cast) integer
2973// value and convert it to the promoted type.
2974SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2975 SDLoc DL(N);
2976
2977 // If the index is constant, try to extract the value from the legalized
2978 // vector type.
2979 if (isa<ConstantSDNode>(N->getOperand(1))) {
2980 SDValue Vec = N->getOperand(0);
2981 SDValue Idx = N->getOperand(1);
2982 EVT VecVT = Vec->getValueType(0);
2983 EVT EltVT = VecVT.getVectorElementType();
2984
2985 uint64_t IdxVal = Idx->getAsZExtVal();
2986
2987 switch (getTypeAction(VecVT)) {
2988 default: break;
2990 SDValue Res = GetScalarizedVector(N->getOperand(0));
2991 ReplaceValueWith(SDValue(N, 0), Res);
2992 return SDValue();
2993 }
2995 Vec = GetWidenedVector(Vec);
2996 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2997 ReplaceValueWith(SDValue(N, 0), Res);
2998 return SDValue();
2999 }
3001 SDValue Lo, Hi;
3002 GetSplitVector(Vec, Lo, Hi);
3003
3004 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
3005 SDValue Res;
3006 if (IdxVal < LoElts)
3007 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
3008 else
3009 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
3010 DAG.getConstant(IdxVal - LoElts, DL,
3011 Idx.getValueType()));
3012 ReplaceValueWith(SDValue(N, 0), Res);
3013 return SDValue();
3014 }
3015
3016 }
3017 }
3018
3019 // Bit-convert the input vector to the equivalent integer vector
3020 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3021 EVT IVT = NewOp.getValueType().getVectorElementType();
3022
3023 // Extract the element as an (bit-cast) integer value
3024 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
3025 NewOp, N->getOperand(1));
3026
3027 // Convert the element to the desired FP type
3028 EVT VT = N->getValueType(0);
3029 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3030 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
3031}
3032
3033// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
3034// needs promotion, so does the argument X. Note that Y, if needed, will be
3035// handled during operand promotion.
3036SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
3037 EVT VT = N->getValueType(0);
3038 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3039 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3040
3041 SDValue Op1 = N->getOperand(1);
3042
3043 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
3044}
3045
3046// Unary operation where the result and the operand have PromoteFloat type
3047// action. Construct a new SDNode with the promoted float value of the old
3048// operand.
3049SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
3050 EVT VT = N->getValueType(0);
3051 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3052 SDValue Op = GetPromotedFloat(N->getOperand(0));
3053 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
3054}
3055
3056// Unary operation with a more non-float operand where the result and the
3057// operand have PromoteFloat type action. Construct a new SDNode with the
3058// promoted float value of the old operand.
3059SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(SDNode *N) {
3060 return GetPromotedFloat(N->getOperand(0));
3061}
3062
3063// Binary operations where the result and both operands have PromoteFloat type
3064// action. Construct a new SDNode with the promoted float values of the old
3065// operands.
3066SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
3067 EVT VT = N->getValueType(0);
3068 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3069 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3070 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
3071 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
3072}
3073
3074SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
3075 EVT VT = N->getValueType(0);
3076 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3077 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3078 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
3079 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
3080
3081 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
3082}
3083
3084// Promote the Float (first) operand and retain the Integer (second) operand
3085SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
3086 EVT VT = N->getValueType(0);
3087 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3088 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3089 SDValue Op1 = N->getOperand(1);
3090
3091 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
3092}
3093
3094SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
3095 EVT VT = N->getValueType(0);
3096 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3097 SDValue Op = GetPromotedFloat(N->getOperand(0));
3098 SDValue Res =
3099 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
3100
3101 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3102 return Res;
3103}
3104
3105SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N) {
3106 EVT VT = N->getValueType(0);
3107 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3108 SDValue Op = GetPromotedFloat(N->getOperand(0));
3109 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op);
3110
3111 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3112 ++ResNum) {
3113 SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum));
3114 }
3115
3116 return SDValue();
3117}
3118
3119// Explicit operation to reduce precision. Reduce the value to half precision
3120// and promote it back to the legal type.
3121SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
3122 SDLoc DL(N);
3123
3124 SDValue Op = N->getOperand(0);
3125 EVT VT = N->getValueType(0);
3126 EVT OpVT = Op->getValueType(0);
3127 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3128 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3129
3130 // Round promoted float to desired precision
3131 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
3132 // Promote it back to the legal output type
3133 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
3134}
3135
3136// Explicit operation to reduce precision. Reduce the value to half precision
3137// and promote it back to the legal type.
3138SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
3139 SDLoc DL(N);
3140
3141 SDValue Chain = N->getOperand(0);
3142 SDValue Op = N->getOperand(1);
3143 EVT VT = N->getValueType(0);
3144 EVT OpVT = Op->getValueType(0);
3145 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3146 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3147
3148 // Round promoted float to desired precision
3149 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
3150 DAG.getVTList(IVT, MVT::Other), Chain, Op);
3151 // Promote it back to the legal output type
3152 SDValue Res =
3153 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
3154 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
3155 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3156 return Res;
3157}
3158
3159SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
3160 LoadSDNode *L = cast<LoadSDNode>(N);
3161 EVT VT = N->getValueType(0);
3162
3163 // Load the value as an integer value with the same number of bits.
3164 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3165 SDValue newL = DAG.getLoad(
3166 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
3167 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
3168 L->getBaseAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
3169 // Legalize the chain result by replacing uses of the old value chain with the
3170 // new one
3171 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
3172
3173 // Convert the integer value to the desired FP type
3174 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3175 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
3176}
3177
3178SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
3179 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3180 EVT VT = AM->getValueType(0);
3181
3182 // Load the value as an integer value with the same number of bits.
3183 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3184 SDValue newL = DAG.getAtomic(
3185 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
3186 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3187
3188 // Legalize the chain result by replacing uses of the old value chain with the
3189 // new one
3190 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
3191
3192 // Convert the integer value to the desired FP type
3193 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3194 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
3195}
3196
3197// Construct a new SELECT node with the promoted true- and false- values.
3198SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
3199 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
3200 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
3201
3202 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
3203 N->getOperand(0), TrueVal, FalseVal);
3204}
3205
3206// Construct a new SELECT_CC node with the promoted true- and false- values.
3207// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
3208SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
3209 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
3210 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
3211
3212 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3213 TrueVal.getNode()->getValueType(0), N->getOperand(0),
3214 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
3215}
3216
3217// Construct a SDNode that transforms the SINT or UINT operand to the promoted
3218// float type.
3219SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
3220 SDLoc DL(N);
3221 EVT VT = N->getValueType(0);
3222 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3223 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
3224 // Round the value to the desired precision (that of the source type).
3225 return DAG.getNode(
3226 ISD::FP_EXTEND, DL, NVT,
3227 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
3228 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
3229}
3230
3231SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
3232 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
3233 N->getValueType(0)));
3234}
3235
3236SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
3237 // Expand and promote recursively.
3238 // TODO: This is non-optimal, but dealing with the concurrently happening
3239 // vector-legalization is non-trivial. We could do something similar to
3240 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
3241 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3242 return SDValue();
3243}
3244
3245SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
3246 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3247 return SDValue();
3248}
3249
3250SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
3251 EVT VT = N->getValueType(0);
3252
3253 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3254 SDLoc SL(N);
3255
3256 SDValue CastVal = BitConvertToInteger(AM->getVal());
3257 EVT CastVT = CastVal.getValueType();
3258
3259 SDValue NewAtomic
3260 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
3261 DAG.getVTList(CastVT, MVT::Other),
3262 { AM->getChain(), AM->getBasePtr(), CastVal },
3263 AM->getMemOperand());
3264
3265 SDValue Result = NewAtomic;
3266
3267 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
3268 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3269 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
3270 NewAtomic);
3271 }
3272
3273 // Legalize the chain result by replacing uses of the old value chain with the
3274 // new one
3275 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
3276
3277 return Result;
3278
3279}
3280
3281//===----------------------------------------------------------------------===//
3282// Half Result Soft Promotion
3283//===----------------------------------------------------------------------===//
3284
3285void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3286 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3287 N->dump(&DAG));
3288 SDValue R = SDValue();
3289
3290 // See if the target wants to custom expand this node.
3291 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
3292 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3293 return;
3294 }
3295
3296 switch (N->getOpcode()) {
3297 default:
3298#ifndef NDEBUG
3299 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3300 N->dump(&DAG); dbgs() << "\n";
3301#endif
3302 report_fatal_error("Do not know how to soft promote this operator's "
3303 "result!");
3304
3305 case ISD::ARITH_FENCE:
3306 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3307 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3308 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3310 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3311 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3313 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3314
3315 // Unary FP Operations
3316 case ISD::FABS:
3317 case ISD::FACOS:
3318 case ISD::FASIN:
3319 case ISD::FATAN:
3320 case ISD::FCBRT:
3321 case ISD::FCEIL:
3322 case ISD::FCOS:
3323 case ISD::FCOSH:
3324 case ISD::FEXP:
3325 case ISD::FEXP2:
3326 case ISD::FEXP10:
3327 case ISD::FFLOOR:
3328 case ISD::FLOG:
3329 case ISD::FLOG2:
3330 case ISD::FLOG10:
3331 case ISD::FNEARBYINT:
3332 case ISD::FNEG:
3333 case ISD::FREEZE:
3334 case ISD::FRINT:
3335 case ISD::FROUND:
3336 case ISD::FROUNDEVEN:
3337 case ISD::FSIN:
3338 case ISD::FSINH:
3339 case ISD::FSQRT:
3340 case ISD::FTRUNC:
3341 case ISD::FTAN:
3342 case ISD::FTANH:
3343 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3345 R = SoftPromoteHalfRes_AssertNoFPClass(N);
3346 break;
3347
3348 // Binary FP Operations
3349 case ISD::FADD:
3350 case ISD::FDIV:
3351 case ISD::FMAXIMUM:
3352 case ISD::FMINIMUM:
3353 case ISD::FMAXIMUMNUM:
3354 case ISD::FMINIMUMNUM:
3355 case ISD::FMAXNUM:
3356 case ISD::FMINNUM:
3357 case ISD::FMUL:
3358 case ISD::FPOW:
3359 case ISD::FATAN2:
3360 case ISD::FREM:
3361 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3362
3363 case ISD::FMA: // FMA is same as FMAD
3364 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3365
3366 case ISD::FPOWI:
3367 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3368
3369 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3370
3371 case ISD::FMODF:
3372 case ISD::FSINCOS:
3373 case ISD::FSINCOSPI:
3374 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
3375 break;
3376
3377 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3378 case ISD::ATOMIC_LOAD:
3379 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3380 break;
3381 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3382 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3385 case ISD::SINT_TO_FP:
3386 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3387 case ISD::POISON:
3388 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3389 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3396 R = SoftPromoteHalfRes_VECREDUCE(N);
3397 break;
3400 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3401 break;
3402 }
3403
3404 if (R.getNode())
3405 SetSoftPromotedHalf(SDValue(N, ResNo), R);
3406}
3407
3408SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3409 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
3410 BitConvertToInteger(N->getOperand(0)));
3411}
3412
3413SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3414 return BitConvertToInteger(N->getOperand(0));
3415}
3416
3417SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3418 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3419
3420 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3421 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3422 MVT::i16);
3423}
3424
3425SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3426 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3428 NewOp.getValueType().getVectorElementType(), NewOp,
3429 N->getOperand(1));
3430}
3431
3432SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3433 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3434 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3435 SDLoc dl(N);
3436
3437 EVT LVT = LHS.getValueType();
3438 EVT RVT = RHS.getValueType();
3439
3440 unsigned LSize = LVT.getSizeInBits();
3441 unsigned RSize = RVT.getSizeInBits();
3442
3443 // First get the sign bit of second operand.
3444 SDValue SignBit = DAG.getNode(
3445 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3446 DAG.getConstant(RSize - 1, dl,
3447 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3448 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3449
3450 // Shift right or sign-extend it if the two operands have different types.
3451 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3452 if (SizeDiff > 0) {
3453 SignBit =
3454 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3455 DAG.getConstant(SizeDiff, dl,
3456 TLI.getShiftAmountTy(SignBit.getValueType(),
3457 DAG.getDataLayout())));
3458 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3459 } else if (SizeDiff < 0) {
3460 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3461 SignBit =
3462 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3463 DAG.getConstant(-SizeDiff, dl,
3464 TLI.getShiftAmountTy(SignBit.getValueType(),
3465 DAG.getDataLayout())));
3466 }
3467
3468 // Clear the sign bit of the first operand.
3469 SDValue Mask = DAG.getNode(
3470 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3471 DAG.getConstant(LSize - 1, dl,
3472 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3473 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3474 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3475
3476 // Or the value with the sign bit.
3477 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3478}
3479
3480SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3481 EVT OVT = N->getValueType(0);
3482 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3483 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3484 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3485 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3486 SDLoc dl(N);
3487
3488 // Promote to the larger FP type.
3489 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3490 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3491 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3492 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3493
3494 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3495
3496 // Convert back to FP16 as an integer.
3497 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3498}
3499
3500SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3501 EVT OVT = N->getValueType(0);
3502 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3503 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3504 SDValue Op1 = N->getOperand(1);
3505 SDLoc dl(N);
3506
3507 // Promote to the larger FP type.
3508 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3509
3510 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3511
3512 // Convert back to FP16 as an integer.
3513 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3514}
3515
3516SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3517 EVT OVT = N->getValueType(0);
3518 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3519 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3520 SDLoc dl(N);
3521
3522 // Promote to the larger FP type.
3523 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3524
3525 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3526 DAG.getVTList(NVT, N->getValueType(1)), Op);
3527
3528 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3529
3530 // Convert back to FP16 as an integer.
3531 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3532}
3533
3534SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
3535 EVT OVT = N->getValueType(0);
3536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3537 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3538 SDLoc dl(N);
3539
3540 // Promote to the larger FP type.
3541 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3542 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, NVT), Op);
3543
3544 // Convert back to FP16 as an integer.
3545 ISD::NodeType Truncate = GetPromotionOpcode(NVT, OVT);
3546 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3547 ++ResNum) {
3548 SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.getValue(ResNum));
3549 SetSoftPromotedHalf(SDValue(N, ResNum), Trunc);
3550 }
3551
3552 return SDValue();
3553}
3554
3555SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3556 EVT RVT = N->getValueType(0);
3557 bool IsStrict = N->isStrictFPOpcode();
3558 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3559 EVT SVT = Op.getValueType();
3560
3561 // If the input type needs to be softened, do that now so that call lowering
3562 // will see the f16 type.
3563 if (getTypeAction(SVT) == TargetLowering::TypeSoftenFloat) {
3564 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, RVT);
3565 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
3566
3567 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3568 Op = GetSoftenedFloat(Op);
3570 CallOptions.setTypeListBeforeSoften(SVT, RVT);
3571 std::pair<SDValue, SDValue> Tmp =
3572 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N), Chain);
3573 if (IsStrict)
3574 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3575 return DAG.getNode(ISD::BITCAST, SDLoc(N), MVT::i16, Tmp.first);
3576 }
3577
3578 if (IsStrict) {
3579 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
3580 {MVT::i16, MVT::Other}, {N->getOperand(0), Op});
3581 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3582 return Res;
3583 }
3584
3585 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3586 N->getOperand(0));
3587}
3588
3589SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3590 LoadSDNode *L = cast<LoadSDNode>(N);
3591
3592 // Load the value as an integer value with the same number of bits.
3593 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3594 SDValue NewL =
3595 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3596 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3597 L->getPointerInfo(), MVT::i16, L->getBaseAlign(),
3598 L->getMemOperand()->getFlags(), L->getAAInfo());
3599 // Legalize the chain result by replacing uses of the old value chain with the
3600 // new one
3601 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3602 return NewL;
3603}
3604
3605SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3606 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3607
3608 // Load the value as an integer value with the same number of bits.
3609 SDValue NewL = DAG.getAtomic(
3610 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3611 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3612
3613 // Legalize the chain result by replacing uses of the old value chain with the
3614 // new one
3615 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3616 return NewL;
3617}
3618
3619SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3620 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3621 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3622 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3623 Op2);
3624}
3625
3626SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3627 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3628 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3629 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3630 N->getOperand(0), N->getOperand(1), Op2, Op3,
3631 N->getOperand(4));
3632}
3633
3634SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3635 EVT OVT = N->getValueType(0);
3636 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3637 SDLoc dl(N);
3638
3639 if (N->isStrictFPOpcode()) {
3640 SDValue Op = DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other},
3641 {N->getOperand(0), N->getOperand(1)});
3642 Op = DAG.getNode(GetPromotionOpcodeStrict(NVT, OVT), dl,
3643 {MVT::i16, MVT::Other}, {Op.getValue(1), Op});
3644 ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
3645 return Op;
3646 }
3647
3648 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3649
3650 // Round the value to the softened type.
3651 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3652}
3653
3654SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3655 return DAG.getUNDEF(MVT::i16);
3656}
3657
3658SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3659 EVT OVT = N->getValueType(0);
3660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3661 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3662 SDLoc dl(N);
3663
3664 // Promote to the larger FP type.
3665 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3666
3667 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3668
3669 // Convert back to FP16 as an integer.
3670 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3671}
3672
3673SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
3674 return GetSoftPromotedHalf(N->getOperand(0));
3675}
3676
3677SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3678 EVT OVT = N->getValueType(0);
3679 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3680 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3681 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3682 SDLoc dl(N);
3683
3684 // Promote to the larger FP type.
3685 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3686 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3687 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3688
3689 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3690
3691 // Convert back to FP16 as an integer.
3692 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3693}
3694
3695SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3696 // Expand and soften recursively.
3697 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3698 return SDValue();
3699}
3700
3701SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3702 // Expand and soften.
3703 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3704 return SDValue();
3705}
3706
3707//===----------------------------------------------------------------------===//
3708// Half Operand Soft Promotion
3709//===----------------------------------------------------------------------===//
3710
3711bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3712 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3713 N->dump(&DAG));
3714 SDValue Res = SDValue();
3715
3716 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3717 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3718 return false;
3719 }
3720
3721 // Nodes that use a promotion-requiring floating point operand, but doesn't
3722 // produce a soft promotion-requiring floating point result, need to be
3723 // legalized to use the soft promoted float operand. Nodes that produce at
3724 // least one soft promotion-requiring floating point result have their
3725 // operands legalized as a part of PromoteFloatResult.
3726 switch (N->getOpcode()) {
3727 default:
3728 #ifndef NDEBUG
3729 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3730 N->dump(&DAG); dbgs() << "\n";
3731 #endif
3732 report_fatal_error("Do not know how to soft promote this operator's "
3733 "operand!");
3734
3735 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3736 case ISD::FAKE_USE:
3737 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3738 break;
3739 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3742 case ISD::FP_TO_SINT:
3743 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3746 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3748 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3749 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3750 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3751 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3752 case ISD::ATOMIC_STORE:
3753 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3754 break;
3755 case ISD::STACKMAP:
3756 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3757 break;
3758 case ISD::PATCHPOINT:
3759 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3760 break;
3761 }
3762
3763 if (!Res.getNode())
3764 return false;
3765
3766 assert(Res.getNode() != N && "Expected a new node!");
3767
3768 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3769 "Invalid operand expansion");
3770
3771 ReplaceValueWith(SDValue(N, 0), Res);
3772 return false;
3773}
3774
3775SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3776 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3777
3778 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3779}
3780
3781SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3782 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3783 SDValue Op = GetSoftPromotedHalf(N->getOperand(OpNo));
3784 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0),
3785 Op);
3786}
3787
3788SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3789 unsigned OpNo) {
3790 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3791 SDValue Op1 = N->getOperand(1);
3792 EVT RVT = Op1.getValueType();
3793 SDLoc dl(N);
3794
3795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3796
3797 Op1 = GetSoftPromotedHalf(Op1);
3798 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3799
3800 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3801 Op1);
3802}
3803
3804SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3805 EVT RVT = N->getValueType(0);
3806 bool IsStrict = N->isStrictFPOpcode();
3807 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3808 EVT SVT = Op.getValueType();
3809 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3810
3811 if (IsStrict) {
3812 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
3813 {RVT, MVT::Other}, {N->getOperand(0), Op});
3814 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3815 ReplaceValueWith(SDValue(N, 0), Res);
3816 return SDValue();
3817 }
3818
3819 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3820}
3821
3822SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3823 EVT RVT = N->getValueType(0);
3824 bool IsStrict = N->isStrictFPOpcode();
3825 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3826 EVT SVT = Op.getValueType();
3827 SDLoc dl(N);
3828
3829 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3830 Op = GetSoftPromotedHalf(Op);
3831
3832 if (IsStrict) {
3833 Op = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), dl, {NVT, MVT::Other},
3834 {N->getOperand(0), Op});
3835 Op = DAG.getNode(N->getOpcode(), dl, {RVT, MVT::Other},
3836 {Op.getValue(1), Op});
3837 ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
3838 ReplaceValueWith(SDValue(N, 0), Op);
3839 return SDValue();
3840 }
3841
3842 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3843 return DAG.getNode(N->getOpcode(), dl, RVT, Res);
3844}
3845
3846SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3847 EVT RVT = N->getValueType(0);
3848 SDValue Op = N->getOperand(0);
3849 EVT SVT = Op.getValueType();
3850 SDLoc dl(N);
3851
3852 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3853
3854 Op = GetSoftPromotedHalf(Op);
3855
3856 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3857
3858 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3859 N->getOperand(1));
3860}
3861
3862SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3863 unsigned OpNo) {
3864 assert(OpNo == 0 && "Can only soften the comparison values");
3865 SDValue Op0 = N->getOperand(0);
3866 SDValue Op1 = N->getOperand(1);
3867 SDLoc dl(N);
3868
3869 EVT SVT = Op0.getValueType();
3870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3871
3872 Op0 = GetSoftPromotedHalf(Op0);
3873 Op1 = GetSoftPromotedHalf(Op1);
3874
3875 // Promote to the larger FP type.
3876 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3877 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3878 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3879
3880 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3881 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3882}
3883
3884SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3885 SDValue Op0 = N->getOperand(0);
3886 SDValue Op1 = N->getOperand(1);
3887 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3888 SDLoc dl(N);
3889
3890 EVT SVT = Op0.getValueType();
3891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3892
3893 Op0 = GetSoftPromotedHalf(Op0);
3894 Op1 = GetSoftPromotedHalf(Op1);
3895
3896 // Promote to the larger FP type.
3897 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3898 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3899 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3900
3901 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3902}
3903
3904SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3905 assert(OpNo == 1 && "Can only soften the stored value!");
3906 StoreSDNode *ST = cast<StoreSDNode>(N);
3907 SDValue Val = ST->getValue();
3908 SDLoc dl(N);
3909
3910 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3911 SDValue Promoted = GetSoftPromotedHalf(Val);
3912 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3913 ST->getMemOperand());
3914}
3915
3916SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3917 unsigned OpNo) {
3918 assert(OpNo == 1 && "Can only soften the stored value!");
3919 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3920 SDValue Val = ST->getVal();
3921 SDLoc dl(N);
3922
3923 SDValue Promoted = GetSoftPromotedHalf(Val);
3924 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3925 ST->getChain(), Promoted, ST->getBasePtr(),
3926 ST->getMemOperand());
3927}
3928
3929SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3930 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3931 SmallVector<SDValue> NewOps(N->ops());
3932 SDValue Op = N->getOperand(OpNo);
3933 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3935 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3936
3937 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3938 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3939
3940 return SDValue(); // Signal that we replaced the node ourselves.
3941}
3942
3943SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3944 unsigned OpNo) {
3945 assert(OpNo >= 7);
3946 SmallVector<SDValue> NewOps(N->ops());
3947 SDValue Op = N->getOperand(OpNo);
3948 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3950 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3951
3952 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3953 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3954
3955 return SDValue(); // Signal that we replaced the node ourselves.
3956}
@ Poison
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
DXIL Intrinsic Expansion
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
#define LLVM_DEBUG(...)
Definition: Debug.h:119
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1353
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:1079
Class for arbitrary precision integers.
Definition: APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1406
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:229
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:569
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:199
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:720
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:498
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:505
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:493
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool expandMultipleResultFPLibCall(RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={})
Expands a node with multiple results to an FP or vector libcall.
LLVMContext * getContext() const
Definition: SelectionDAG.h:511
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:581
void push_back(const T &Elt)
Definition: SmallVector.h:414
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1197
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:126
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:41
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:801
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:256
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:504
@ POISON
POISON - A poison node.
Definition: ISDOpcodes.h:231
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1458
@ ConstantFP
Definition: ISDOpcodes.h:87
@ STRICT_FATAN2
Definition: ISDOpcodes.h:441
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1351
@ STRICT_FCEIL
Definition: ISDOpcodes.h:454
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:515
@ STRICT_FTANH
Definition: ISDOpcodes.h:444
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1141
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:835
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:511
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
Definition: ISDOpcodes.h:1098
@ FATAN2
FATAN2 - atan2, inspired by libm.
Definition: ISDOpcodes.h:1020
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
Definition: ISDOpcodes.h:1094
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:862
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1476
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:410
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1480
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:505
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:985
@ STRICT_FLOG2
Definition: ISDOpcodes.h:449
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
Definition: ISDOpcodes.h:1424
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:975
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:249
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:1018
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:431
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:826
@ STRICT_FASIN
Definition: ISDOpcodes.h:438
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:478
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1473
@ STRICT_FATAN
Definition: ISDOpcodes.h:440
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1477
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:1090
@ STRICT_LROUND
Definition: ISDOpcodes.h:459
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:1002
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1187
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:528
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:778
@ STRICT_FPOWI
Definition: ISDOpcodes.h:433
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1347
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:228
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:242
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:458
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1335
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:988
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:987
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:756
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition: ISDOpcodes.h:78
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:1075
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:452
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:563
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:832
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:928
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:453
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:793
@ STRICT_FSINH
Definition: ISDOpcodes.h:442
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum maximum on two values, following IEEE-754 definition...
Definition: ISDOpcodes.h:1059
@ STRICT_LRINT
Definition: ISDOpcodes.h:461
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:960
@ STRICT_FROUND
Definition: ISDOpcodes.h:456
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:477
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:996
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:455
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:457
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:994
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:471
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:493
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:470
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1081
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:908
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:498
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:730
@ STRICT_FCOSH
Definition: ISDOpcodes.h:443
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:997
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1474
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:420
@ STRICT_FLOG10
Definition: ISDOpcodes.h:448
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:236
@ STRICT_LLRINT
Definition: ISDOpcodes.h:462
@ STRICT_FEXP2
Definition: ISDOpcodes.h:446
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1372
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1025
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:941
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:434
@ STRICT_LLROUND
Definition: ISDOpcodes.h:460
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:451
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:927
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1481
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:838
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1256
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1459
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:521
@ STRICT_FRINT
Definition: ISDOpcodes.h:450
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
Definition: ISDOpcodes.h:1086
@ STRICT_FACOS
Definition: ISDOpcodes.h:439
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:543
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1691
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOSPI(EVT RetVT)
getSINCOSPI - Return the SINCOSPI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getMODF(EVT RetVT)
getMODF - Return the MODF_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getCOS(EVT RetVT)
Return the COS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSIN(EVT RetVT)
Return the SIN_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getPOW(EVT RetVT)
getPOW - Return the POW_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOS(EVT RetVT)
getSINCOS - Return the SINCOS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:477
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2491
void * PointerTy
Definition: GenericValue.h:21
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:207
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition: Error.cpp:167
DWARFExpression::Operation Op
#define N
static LLVM_ABI const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:269
Extended Value Type.
Definition: ValueTypes.h:35
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:137
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:368
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:238
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:311
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:287
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:323
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
Definition: ValueTypes.cpp:330
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:303
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)