LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::STRICT_FCEIL:
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::STRICT_FCOS:
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::STRICT_FDIV:
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::STRICT_FEXP:
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::STRICT_FEXP2:
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::STRICT_FFLOOR:
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::STRICT_FLOG:
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::STRICT_FLOG2:
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::STRICT_FLOG10:
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::STRICT_FNEARBYINT:
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::STRICT_FP_EXTEND:
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::STRICT_FP_ROUND:
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::STRICT_FPOWI:
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
122 case ISD::STRICT_FRINT:
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
124 case ISD::STRICT_FROUND:
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
126 case ISD::STRICT_FROUNDEVEN:
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
130 case ISD::STRICT_FSQRT:
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
134 case ISD::STRICT_FTAN:
135 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
136 case ISD::STRICT_FTRUNC:
137 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
138 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
139 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
140 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
141 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
142 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
143 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
144 case ISD::STRICT_SINT_TO_FP:
145 case ISD::STRICT_UINT_TO_FP:
146 case ISD::SINT_TO_FP:
147 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
148 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
149 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
150 case ISD::VECREDUCE_FADD:
151 case ISD::VECREDUCE_FMUL:
152 case ISD::VECREDUCE_FMIN:
153 case ISD::VECREDUCE_FMAX:
154 case ISD::VECREDUCE_FMAXIMUM:
155 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
156 case ISD::VECREDUCE_SEQ_FADD:
157 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
158 // clang-format on
159 }
160
161 // If R is null, the sub-method took care of registering the result.
162 if (R.getNode()) {
163 assert(R.getNode() != N);
164 SetSoftenedFloat(SDValue(N, ResNo), R);
165 }
166}
167
168SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
169 bool IsStrict = N->isStrictFPOpcode();
170 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
171 unsigned Offset = IsStrict ? 1 : 0;
172 assert(N->getNumOperands() == (1 + Offset) &&
173 "Unexpected number of operands!");
174 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
175 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
176 TargetLowering::MakeLibCallOptions CallOptions;
177 EVT OpVT = N->getOperand(0 + Offset).getValueType();
178 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
179 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
180 CallOptions, SDLoc(N),
181 Chain);
182 if (IsStrict)
183 ReplaceValueWith(SDValue(N, 1), Tmp.second);
184 return Tmp.first;
185}
186
187SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
188 bool IsStrict = N->isStrictFPOpcode();
189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
190 unsigned Offset = IsStrict ? 1 : 0;
191 assert(N->getNumOperands() == (2 + Offset) &&
192 "Unexpected number of operands!");
193 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
194 GetSoftenedFloat(N->getOperand(1 + Offset)) };
195 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
196 TargetLowering::MakeLibCallOptions CallOptions;
197 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
198 N->getOperand(1 + Offset).getValueType() };
199 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
200 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
201 CallOptions, SDLoc(N),
202 Chain);
203 if (IsStrict)
204 ReplaceValueWith(SDValue(N, 1), Tmp.second);
205 return Tmp.first;
206}
207
208SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
209 return BitConvertToInteger(N->getOperand(0));
210}
211
212SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
213 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
214 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
215 GetSoftenedFloat(N->getOperand(0)));
216}
217
218SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
219 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
220 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
221 GetSoftenedFloat(N->getOperand(0)));
222 return NewFence;
223}
224
225SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
226 unsigned ResNo) {
227 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
228 return BitConvertToInteger(Op);
229}
230
231SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
232 // Convert the inputs to integers, and build a new pair out of them.
233 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
234 TLI.getTypeToTransformTo(*DAG.getContext(),
235 N->getValueType(0)),
236 BitConvertToInteger(N->getOperand(0)),
237 BitConvertToInteger(N->getOperand(1)));
238}
239
240SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
241 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
242 // In ppcf128, the high 64 bits are always first in memory regardless
243 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
244 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
245 // way. However, APInt's are serialized in an Endian-sensitive fashion,
246 // so on big-Endian targets, the two doubles are output in the wrong
247 // order. Fix this by manually flipping the order of the high 64 bits
248 // and the low 64 bits here.
249 if (DAG.getDataLayout().isBigEndian() &&
250 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
251 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
252 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
253 APInt Val(128, words);
254 return DAG.getConstant(Val, SDLoc(CN),
255 TLI.getTypeToTransformTo(*DAG.getContext(),
256 CN->getValueType(0)));
257 } else {
258 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
259 TLI.getTypeToTransformTo(*DAG.getContext(),
260 CN->getValueType(0)));
261 }
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
265 SDValue Src = N->getOperand(0);
266 assert(Src.getValueType() == MVT::ppcf128 &&
267 "In floats only ppcf128 can be extracted by element!");
268 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
269 N->getValueType(0).changeTypeToInteger(),
270 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
271}
272
273SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
274 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
275 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
276 NewOp.getValueType().getVectorElementType(),
277 NewOp, N->getOperand(1));
278}
279
280SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
281 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
282 unsigned Size = NVT.getSizeInBits();
283
284 // Mask = ~(1 << (Size-1))
285 APInt API = APInt::getAllOnes(Size);
286 API.clearBit(Size - 1);
287 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
288 SDValue Op = GetSoftenedFloat(N->getOperand(0));
289 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
290}
291
292SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
293 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
294 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
295 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
296 RTLIB::FMIN_F32,
297 RTLIB::FMIN_F64,
298 RTLIB::FMIN_F80,
299 RTLIB::FMIN_F128,
300 RTLIB::FMIN_PPCF128));
301}
302
303SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
304 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
305 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
306 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
307 RTLIB::FMAX_F32,
308 RTLIB::FMAX_F64,
309 RTLIB::FMAX_F80,
310 RTLIB::FMAX_F128,
311 RTLIB::FMAX_PPCF128));
312}
313
314SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
315 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
316 RTLIB::ADD_F32,
317 RTLIB::ADD_F64,
318 RTLIB::ADD_F80,
319 RTLIB::ADD_F128,
320 RTLIB::ADD_PPCF128));
321}
322
323SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
324 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
325 RTLIB::CBRT_F32,
326 RTLIB::CBRT_F64,
327 RTLIB::CBRT_F80,
328 RTLIB::CBRT_F128,
329 RTLIB::CBRT_PPCF128));
330}
331
332SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
333 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
334 RTLIB::CEIL_F32,
335 RTLIB::CEIL_F64,
336 RTLIB::CEIL_F80,
337 RTLIB::CEIL_F128,
338 RTLIB::CEIL_PPCF128));
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
342 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
343 SDValue RHS = BitConvertToInteger(N->getOperand(1));
344 SDLoc dl(N);
345
346 EVT LVT = LHS.getValueType();
347 EVT RVT = RHS.getValueType();
348
349 unsigned LSize = LVT.getSizeInBits();
350 unsigned RSize = RVT.getSizeInBits();
351
352 // First get the sign bit of second operand.
353 SDValue SignBit = DAG.getNode(
354 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
355 DAG.getConstant(RSize - 1, dl,
356 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
357 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
358
359 // Shift right or sign-extend it if the two operands have different types.
360 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
361 if (SizeDiff > 0) {
362 SignBit =
363 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
364 DAG.getConstant(SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
368 } else if (SizeDiff < 0) {
369 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
370 SignBit =
371 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
372 DAG.getConstant(-SizeDiff, dl,
373 TLI.getShiftAmountTy(SignBit.getValueType(),
374 DAG.getDataLayout())));
375 }
376
377 // Clear the sign bit of the first operand.
378 SDValue Mask = DAG.getNode(
379 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
380 DAG.getConstant(LSize - 1, dl,
381 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
382 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
383 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
384
385 // Or the value with the sign bit.
386 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
387}
388
389SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
390 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
391 RTLIB::COS_F32,
392 RTLIB::COS_F64,
393 RTLIB::COS_F80,
394 RTLIB::COS_F128,
395 RTLIB::COS_PPCF128));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
399 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
400 RTLIB::DIV_F32,
401 RTLIB::DIV_F64,
402 RTLIB::DIV_F80,
403 RTLIB::DIV_F128,
404 RTLIB::DIV_PPCF128));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
408 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
409 RTLIB::EXP_F32,
410 RTLIB::EXP_F64,
411 RTLIB::EXP_F80,
412 RTLIB::EXP_F128,
413 RTLIB::EXP_PPCF128));
414}
415
416SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
417 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
418 RTLIB::EXP2_F32,
419 RTLIB::EXP2_F64,
420 RTLIB::EXP2_F80,
421 RTLIB::EXP2_F128,
422 RTLIB::EXP2_PPCF128));
423}
424
425SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
426 return SoftenFloatRes_Unary(
427 N,
428 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
429 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
430}
431
432SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
433 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
434 RTLIB::FLOOR_F32,
435 RTLIB::FLOOR_F64,
436 RTLIB::FLOOR_F80,
437 RTLIB::FLOOR_F128,
438 RTLIB::FLOOR_PPCF128));
439}
440
441SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
442 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
443 RTLIB::LOG_F32,
444 RTLIB::LOG_F64,
445 RTLIB::LOG_F80,
446 RTLIB::LOG_F128,
447 RTLIB::LOG_PPCF128));
448}
449
450SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
451 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
452 RTLIB::LOG2_F32,
453 RTLIB::LOG2_F64,
454 RTLIB::LOG2_F80,
455 RTLIB::LOG2_F128,
456 RTLIB::LOG2_PPCF128));
457}
458
459SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
460 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
461 RTLIB::LOG10_F32,
462 RTLIB::LOG10_F64,
463 RTLIB::LOG10_F80,
464 RTLIB::LOG10_F128,
465 RTLIB::LOG10_PPCF128));
466}
467
468SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
469 bool IsStrict = N->isStrictFPOpcode();
470 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
471 unsigned Offset = IsStrict ? 1 : 0;
472 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
473 GetSoftenedFloat(N->getOperand(1 + Offset)),
474 GetSoftenedFloat(N->getOperand(2 + Offset)) };
475 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
476 TargetLowering::MakeLibCallOptions CallOptions;
477 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
478 N->getOperand(1 + Offset).getValueType(),
479 N->getOperand(2 + Offset).getValueType() };
480 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
481 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
482 GetFPLibCall(N->getValueType(0),
483 RTLIB::FMA_F32,
484 RTLIB::FMA_F64,
485 RTLIB::FMA_F80,
486 RTLIB::FMA_F128,
487 RTLIB::FMA_PPCF128),
488 NVT, Ops, CallOptions, SDLoc(N), Chain);
489 if (IsStrict)
490 ReplaceValueWith(SDValue(N, 1), Tmp.second);
491 return Tmp.first;
492}
493
494SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
495 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
496 RTLIB::MUL_F32,
497 RTLIB::MUL_F64,
498 RTLIB::MUL_F80,
499 RTLIB::MUL_F128,
500 RTLIB::MUL_PPCF128));
501}
502
503SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
504 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
505 RTLIB::NEARBYINT_F32,
506 RTLIB::NEARBYINT_F64,
507 RTLIB::NEARBYINT_F80,
508 RTLIB::NEARBYINT_F128,
509 RTLIB::NEARBYINT_PPCF128));
510}
511
512SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
513 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
514 SDLoc dl(N);
515
516 // Expand Y = FNEG(X) -> Y = X ^ sign mask
517 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
518 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
519 DAG.getConstant(SignMask, dl, NVT));
520}
521
522SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
523 bool IsStrict = N->isStrictFPOpcode();
524 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
525 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
526
527 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
528
529 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
530 Op = GetPromotedFloat(Op);
531 // If the promotion did the FP_EXTEND to the destination type for us,
532 // there's nothing left to do here.
533 if (Op.getValueType() == N->getValueType(0)) {
534 if (IsStrict)
535 ReplaceValueWith(SDValue(N, 1), Chain);
536 return BitConvertToInteger(Op);
537 }
538 }
539
540 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
541 // -> f32, so proceed in two stages. Also, it's entirely possible for both
542 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
543 // than FP16_TO_FP.
544 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
545 N->getValueType(0) != MVT::f32) {
546 if (IsStrict) {
547 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
548 { MVT::f32, MVT::Other }, { Chain, Op });
549 Chain = Op.getValue(1);
550 } else {
551 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
552 }
553 }
554
555 if (Op.getValueType() == MVT::bf16) {
556 // FIXME: Need ReplaceValueWith on chain in strict case
557 return SoftenFloatRes_BF16_TO_FP(N);
558 }
559
560 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
561 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
562 TargetLowering::MakeLibCallOptions CallOptions;
563 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
564 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
565 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
566 CallOptions, SDLoc(N),
567 Chain);
568 if (IsStrict)
569 ReplaceValueWith(SDValue(N, 1), Tmp.second);
570 return Tmp.first;
571}
572
573// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
574// nodes?
575SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
576 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
577 SDValue Op = N->getOperand(0);
578 TargetLowering::MakeLibCallOptions CallOptions;
579 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
580 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
581 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
582 CallOptions, SDLoc(N)).first;
583 if (N->getValueType(0) == MVT::f32)
584 return Res32;
585
586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
587 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
588 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
589 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
590}
591
592// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
593// nodes?
594SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
595 assert(N->getValueType(0) == MVT::f32 &&
596 "Can only soften BF16_TO_FP with f32 result");
597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
598 SDValue Op = N->getOperand(0);
599 SDLoc DL(N);
600 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
601 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
602 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
603 DAG.getShiftAmountConstant(16, NVT, DL));
604 return Res;
605}
606
607SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
608 bool IsStrict = N->isStrictFPOpcode();
609 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
610 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
611 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
612 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
613 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
614 TargetLowering::MakeLibCallOptions CallOptions;
615 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
616 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
617 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
618 CallOptions, SDLoc(N),
619 Chain);
620 if (IsStrict)
621 ReplaceValueWith(SDValue(N, 1), Tmp.second);
622 return Tmp.first;
623}
624
625SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
626 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
627 RTLIB::POW_F32,
628 RTLIB::POW_F64,
629 RTLIB::POW_F80,
630 RTLIB::POW_F128,
631 RTLIB::POW_PPCF128));
632}
633
634SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
635 bool IsStrict = N->isStrictFPOpcode();
636 unsigned Offset = IsStrict ? 1 : 0;
637 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
638 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
639 "Unsupported power type!");
640 bool IsPowI =
641 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
642
643 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
644 : RTLIB::getLDEXP(N->getValueType(0));
645 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
646 if (!TLI.getLibcallName(LC)) {
647 // Some targets don't have a powi libcall; use pow instead.
648 // FIXME: Implement this if some target needs it.
649 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
650 return DAG.getUNDEF(N->getValueType(0));
651 }
652
653 if (DAG.getLibInfo().getIntSize() !=
654 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
655 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
656 // would use the wrong type for the argument.
657 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
658 return DAG.getUNDEF(N->getValueType(0));
659 }
660
661 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
662 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
663 N->getOperand(1 + Offset) };
664 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
665 TargetLowering::MakeLibCallOptions CallOptions;
666 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
667 N->getOperand(1 + Offset).getValueType() };
668 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
669 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
670 CallOptions, SDLoc(N),
671 Chain);
672 if (IsStrict)
673 ReplaceValueWith(SDValue(N, 1), Tmp.second);
674 return Tmp.first;
675}
676
677SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
678 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
679 EVT VT0 = N->getValueType(0);
680 EVT VT1 = N->getValueType(1);
681 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
682
683 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
684 // If the exponent does not match with sizeof(int) a libcall would use the
685 // wrong type for the argument.
686 // TODO: Should be able to handle mismatches.
687 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
688 return DAG.getUNDEF(N->getValueType(0));
689 }
690
691 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
692 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
693
694 SDLoc DL(N);
695
696 TargetLowering::MakeLibCallOptions CallOptions;
697 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
698 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
699
700 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
701 // but we only really need to handle the 0th one for softening anyway.
702 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
703
704 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
705 /*Chain=*/SDValue());
706 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
707 auto PtrInfo =
708 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
709
710 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
711
712 ReplaceValueWith(SDValue(N, 1), LoadExp);
713 return ReturnVal;
714}
715
716SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
717 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
718 RTLIB::REM_F32,
719 RTLIB::REM_F64,
720 RTLIB::REM_F80,
721 RTLIB::REM_F128,
722 RTLIB::REM_PPCF128));
723}
724
725SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
726 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
727 RTLIB::RINT_F32,
728 RTLIB::RINT_F64,
729 RTLIB::RINT_F80,
730 RTLIB::RINT_F128,
731 RTLIB::RINT_PPCF128));
732}
733
734SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
735 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
736 RTLIB::ROUND_F32,
737 RTLIB::ROUND_F64,
738 RTLIB::ROUND_F80,
739 RTLIB::ROUND_F128,
740 RTLIB::ROUND_PPCF128));
741}
742
743SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
744 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
745 RTLIB::ROUNDEVEN_F32,
746 RTLIB::ROUNDEVEN_F64,
747 RTLIB::ROUNDEVEN_F80,
748 RTLIB::ROUNDEVEN_F128,
749 RTLIB::ROUNDEVEN_PPCF128));
750}
751
752SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
753 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
754 RTLIB::SIN_F32,
755 RTLIB::SIN_F64,
756 RTLIB::SIN_F80,
757 RTLIB::SIN_F128,
758 RTLIB::SIN_PPCF128));
759}
760
761SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
762 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
763 RTLIB::SQRT_F32,
764 RTLIB::SQRT_F64,
765 RTLIB::SQRT_F80,
766 RTLIB::SQRT_F128,
767 RTLIB::SQRT_PPCF128));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
771 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
772 RTLIB::SUB_F32,
773 RTLIB::SUB_F64,
774 RTLIB::SUB_F80,
775 RTLIB::SUB_F128,
776 RTLIB::SUB_PPCF128));
777}
778
779SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
780 return SoftenFloatRes_Unary(
781 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
782 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
783}
784
785SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
786 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
787 RTLIB::TRUNC_F32,
788 RTLIB::TRUNC_F64,
789 RTLIB::TRUNC_F80,
790 RTLIB::TRUNC_F128,
791 RTLIB::TRUNC_PPCF128));
792}
793
794SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
795 LoadSDNode *L = cast<LoadSDNode>(N);
796 EVT VT = N->getValueType(0);
797 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
798 SDLoc dl(N);
799
800 auto MMOFlags =
801 L->getMemOperand()->getFlags() &
802 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
803 SDValue NewL;
804 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
805 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
806 L->getChain(), L->getBasePtr(), L->getOffset(),
807 L->getPointerInfo(), NVT, L->getOriginalAlign(),
808 MMOFlags, L->getAAInfo());
809 // Legalized the chain result - switch anything that used the old chain to
810 // use the new one.
811 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
812 return NewL;
813 }
814
815 // Do a non-extending load followed by FP_EXTEND.
816 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
817 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
818 L->getPointerInfo(), L->getMemoryVT(),
819 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
820 // Legalized the chain result - switch anything that used the old chain to
821 // use the new one.
822 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
823 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
824 return BitConvertToInteger(ExtendNode);
825}
826
827SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
828 AtomicSDNode *L = cast<AtomicSDNode>(N);
829 EVT VT = N->getValueType(0);
830 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
831 SDLoc dl(N);
832
833 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
834 SDValue NewL =
835 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
836 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
837
838 // Legalized the chain result - switch anything that used the old chain to
839 // use the new one.
840 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
841 return NewL;
842 }
843
844 report_fatal_error("softening fp extending atomic load not handled");
845}
846
847SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
848 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
849 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
850 return DAG.getSelect(SDLoc(N),
851 LHS.getValueType(), N->getOperand(0), LHS, RHS);
852}
853
854SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
855 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
856 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
857 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
858 LHS.getValueType(), N->getOperand(0),
859 N->getOperand(1), LHS, RHS, N->getOperand(4));
860}
861
862SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
863 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
864 N->getValueType(0)));
865}
866
867SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
868 SDValue Chain = N->getOperand(0); // Get the chain.
869 SDValue Ptr = N->getOperand(1); // Get the pointer.
870 EVT VT = N->getValueType(0);
871 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
872 SDLoc dl(N);
873
874 SDValue NewVAARG;
875 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
876 N->getConstantOperandVal(3));
877
878 // Legalized the chain result - switch anything that used the old chain to
879 // use the new one.
880 if (N != NewVAARG.getValue(1).getNode())
881 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
882 return NewVAARG;
883}
884
885SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
886 bool IsStrict = N->isStrictFPOpcode();
887 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
888 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
889 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
890 EVT RVT = N->getValueType(0);
891 EVT NVT = EVT();
892 SDLoc dl(N);
893
894 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
895 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
896 // match. Look for an appropriate libcall.
897 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
898 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
899 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
900 NVT = (MVT::SimpleValueType)t;
901 // The source needs to big enough to hold the operand.
902 if (NVT.bitsGE(SVT))
903 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
904 }
905 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
906
907 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
908 // Sign/zero extend the argument if the libcall takes a larger type.
909 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
910 NVT, N->getOperand(IsStrict ? 1 : 0));
911 TargetLowering::MakeLibCallOptions CallOptions;
912 CallOptions.setSExt(Signed);
913 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
914 std::pair<SDValue, SDValue> Tmp =
915 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
916 Op, CallOptions, dl, Chain);
917
918 if (IsStrict)
919 ReplaceValueWith(SDValue(N, 1), Tmp.second);
920 return Tmp.first;
921}
922
923SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
924 // Expand and soften recursively.
925 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
926 return SDValue();
927}
928
929SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
930 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
931 return SDValue();
932}
933
934//===----------------------------------------------------------------------===//
935// Convert Float Operand to Integer
936//===----------------------------------------------------------------------===//
937
938bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
939 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
940 SDValue Res = SDValue();
941
942 switch (N->getOpcode()) {
943 default:
944#ifndef NDEBUG
945 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
946 N->dump(&DAG); dbgs() << "\n";
947#endif
948 report_fatal_error("Do not know how to soften this operator's operand!");
949
950 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
951 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
952 case ISD::STRICT_FP_TO_FP16:
953 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
954 case ISD::FP_TO_BF16:
955 case ISD::STRICT_FP_TO_BF16:
956 case ISD::STRICT_FP_ROUND:
957 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
958 case ISD::STRICT_FP_TO_SINT:
959 case ISD::STRICT_FP_TO_UINT:
960 case ISD::FP_TO_SINT:
961 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
962 case ISD::FP_TO_SINT_SAT:
963 case ISD::FP_TO_UINT_SAT:
964 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
965 case ISD::STRICT_LROUND:
966 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
967 case ISD::STRICT_LLROUND:
968 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
969 case ISD::STRICT_LRINT:
970 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
971 case ISD::STRICT_LLRINT:
972 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
973 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
974 case ISD::STRICT_FSETCC:
975 case ISD::STRICT_FSETCCS:
976 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
977 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
978 case ISD::ATOMIC_STORE:
979 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
980 break;
981 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
982 }
983
984 // If the result is null, the sub-method took care of registering results etc.
985 if (!Res.getNode()) return false;
986
987 // If the result is N, the sub-method updated N in place. Tell the legalizer
988 // core about this to re-analyze.
989 if (Res.getNode() == N)
990 return true;
991
992 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
993 "Invalid operand softening");
994
995 ReplaceValueWith(SDValue(N, 0), Res);
996 return false;
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1000 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1001
1002 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1003}
1004
1005SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1006 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1007 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1008 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1009 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1010 N->getOpcode() == ISD::FP_TO_BF16 ||
1011 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1012 N->getOpcode() == ISD::STRICT_FP_ROUND);
1013
1014 bool IsStrict = N->isStrictFPOpcode();
1015 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1016 EVT SVT = Op.getValueType();
1017 EVT RVT = N->getValueType(0);
1018 EVT FloatRVT = RVT;
1019 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1020 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1021 FloatRVT = MVT::f16;
1022 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1023 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1024 FloatRVT = MVT::bf16;
1025
1026 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1027 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1028
1029 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1030 Op = GetSoftenedFloat(Op);
1031 TargetLowering::MakeLibCallOptions CallOptions;
1032 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1033 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1034 CallOptions, SDLoc(N),
1035 Chain);
1036 if (IsStrict) {
1037 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1038 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1039 return SDValue();
1040 }
1041 return Tmp.first;
1042}
1043
1044SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1045 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1046 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1047
1048 EVT VT = NewLHS.getValueType();
1049 NewLHS = GetSoftenedFloat(NewLHS);
1050 NewRHS = GetSoftenedFloat(NewRHS);
1051 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1052 N->getOperand(2), N->getOperand(3));
1053
1054 // If softenSetCCOperands returned a scalar, we need to compare the result
1055 // against zero to select between true and false values.
1056 if (!NewRHS.getNode()) {
1057 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1058 CCCode = ISD::SETNE;
1059 }
1060
1061 // Update N to have the operands specified.
1062 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1063 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1064 N->getOperand(4)),
1065 0);
1066}
1067
1068// Even if the result type is legal, no libcall may exactly match. (e.g. We
1069// don't have FP-i8 conversions) This helper method looks for an appropriate
1070// promoted libcall.
1071static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1072 bool Signed) {
1073 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1074 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1075 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1076 ++IntVT) {
1077 Promoted = (MVT::SimpleValueType)IntVT;
1078 // The type needs to big enough to hold the result.
1079 if (Promoted.bitsGE(RetVT))
1080 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1081 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1082 }
1083 return LC;
1084}
1085
1086SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1087 bool IsStrict = N->isStrictFPOpcode();
1088 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1089 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1090
1091 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1092 EVT SVT = Op.getValueType();
1093 EVT RVT = N->getValueType(0);
1094 EVT NVT = EVT();
1095 SDLoc dl(N);
1096
1097 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1098 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1099 // match, eg. we don't have fp -> i8 conversions.
1100 // Look for an appropriate libcall.
1101 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1102 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1103 "Unsupported FP_TO_XINT!");
1104
1105 Op = GetSoftenedFloat(Op);
1106 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1107 TargetLowering::MakeLibCallOptions CallOptions;
1108 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1109 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1110 CallOptions, dl, Chain);
1111
1112 // Truncate the result if the libcall returns a larger type.
1113 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1114
1115 if (!IsStrict)
1116 return Res;
1117
1118 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1119 ReplaceValueWith(SDValue(N, 0), Res);
1120 return SDValue();
1121}
1122
1123SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1124 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1125 return Res;
1126}
1127
1128SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1129 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1130 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1131
1132 EVT VT = NewLHS.getValueType();
1133 NewLHS = GetSoftenedFloat(NewLHS);
1134 NewRHS = GetSoftenedFloat(NewRHS);
1135 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1136 N->getOperand(0), N->getOperand(1));
1137
1138 // If softenSetCCOperands returned a scalar, we need to compare the result
1139 // against zero to select between true and false values.
1140 if (!NewRHS.getNode()) {
1141 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1142 CCCode = ISD::SETNE;
1143 }
1144
1145 // Update N to have the operands specified.
1146 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1147 N->getOperand(2), N->getOperand(3),
1148 DAG.getCondCode(CCCode)),
1149 0);
1150}
1151
1152SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1153 bool IsStrict = N->isStrictFPOpcode();
1154 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1155 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1156 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1157 ISD::CondCode CCCode =
1158 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1159
1160 EVT VT = Op0.getValueType();
1161 SDValue NewLHS = GetSoftenedFloat(Op0);
1162 SDValue NewRHS = GetSoftenedFloat(Op1);
1163 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1164 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1165
1166 // Update N to have the operands specified.
1167 if (NewRHS.getNode()) {
1168 if (IsStrict)
1169 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1170 NewRHS, DAG.getCondCode(CCCode));
1171 else
1172 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1173 DAG.getCondCode(CCCode)), 0);
1174 }
1175
1176 // Otherwise, softenSetCCOperands returned a scalar, use it.
1177 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1178 "Unexpected setcc expansion!");
1179
1180 if (IsStrict) {
1181 ReplaceValueWith(SDValue(N, 0), NewLHS);
1182 ReplaceValueWith(SDValue(N, 1), Chain);
1183 return SDValue();
1184 }
1185 return NewLHS;
1186}
1187
1188SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1189 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1190 assert(OpNo == 1 && "Can only soften the stored value!");
1191 StoreSDNode *ST = cast<StoreSDNode>(N);
1192 SDValue Val = ST->getValue();
1193 SDLoc dl(N);
1194
1195 if (ST->isTruncatingStore())
1196 // Do an FP_ROUND followed by a non-truncating store.
1197 Val = BitConvertToInteger(
1198 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1199 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1200 else
1201 Val = GetSoftenedFloat(Val);
1202
1203 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1204 ST->getMemOperand());
1205}
1206
1207SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1208 assert(OpNo == 1 && "Can only soften the stored value!");
1209 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1210 SDValue Val = ST->getVal();
1211 EVT VT = Val.getValueType();
1212 SDLoc dl(N);
1213
1214 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1215
1216 SDValue NewVal = GetSoftenedFloat(Val);
1217 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1218 ST->getBasePtr(), ST->getMemOperand());
1219}
1220
1221SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1222 SDValue LHS = N->getOperand(0);
1223 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1224 SDLoc dl(N);
1225
1226 EVT LVT = LHS.getValueType();
1227 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1228 EVT RVT = RHS.getValueType();
1229
1230 unsigned LSize = LVT.getSizeInBits();
1231 unsigned RSize = RVT.getSizeInBits();
1232
1233 // Shift right or sign-extend it if the two operands have different types.
1234 int SizeDiff = RSize - LSize;
1235 if (SizeDiff > 0) {
1236 RHS =
1237 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1238 DAG.getConstant(SizeDiff, dl,
1239 TLI.getShiftAmountTy(RHS.getValueType(),
1240 DAG.getDataLayout())));
1241 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1242 } else if (SizeDiff < 0) {
1243 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1244 RHS =
1245 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1246 DAG.getConstant(-SizeDiff, dl,
1247 TLI.getShiftAmountTy(RHS.getValueType(),
1248 DAG.getDataLayout())));
1249 }
1250
1251 RHS = DAG.getBitcast(LVT, RHS);
1252 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1253}
1254
1255SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1257 bool IsStrict = N->isStrictFPOpcode();
1258 unsigned Offset = IsStrict ? 1 : 0;
1259 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1260 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1261 TargetLowering::MakeLibCallOptions CallOptions;
1262 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1263 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1264 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1265 CallOptions, SDLoc(N),
1266 Chain);
1267 if (IsStrict) {
1268 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1269 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1270 return SDValue();
1271 }
1272
1273 return Tmp.first;
1274}
1275
1276SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1277 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1278 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1279 RTLIB::LROUND_F32,
1280 RTLIB::LROUND_F64,
1281 RTLIB::LROUND_F80,
1282 RTLIB::LROUND_F128,
1283 RTLIB::LROUND_PPCF128));
1284}
1285
1286SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1287 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1288 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1289 RTLIB::LLROUND_F32,
1290 RTLIB::LLROUND_F64,
1291 RTLIB::LLROUND_F80,
1292 RTLIB::LLROUND_F128,
1293 RTLIB::LLROUND_PPCF128));
1294}
1295
1296SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1297 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1298 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1299 RTLIB::LRINT_F32,
1300 RTLIB::LRINT_F64,
1301 RTLIB::LRINT_F80,
1302 RTLIB::LRINT_F128,
1303 RTLIB::LRINT_PPCF128));
1304}
1305
1306SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1307 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1308 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1309 RTLIB::LLRINT_F32,
1310 RTLIB::LLRINT_F64,
1311 RTLIB::LLRINT_F80,
1312 RTLIB::LLRINT_F128,
1313 RTLIB::LLRINT_PPCF128));
1314}
1315
1316//===----------------------------------------------------------------------===//
1317// Float Result Expansion
1318//===----------------------------------------------------------------------===//
1319
1320/// ExpandFloatResult - This method is called when the specified result of the
1321/// specified node is found to need expansion. At this point, the node may also
1322/// have invalid operands or may have other results that need promotion, we just
1323/// know that (at least) one result needs expansion.
1324void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1325 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1326 SDValue Lo, Hi;
1327 Lo = Hi = SDValue();
1328
1329 // See if the target wants to custom expand this node.
1330 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1331 return;
1332
1333 switch (N->getOpcode()) {
1334 default:
1335#ifndef NDEBUG
1336 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1337 N->dump(&DAG); dbgs() << "\n";
1338#endif
1339 report_fatal_error("Do not know how to expand the result of this "
1340 "operator!");
1341 // clang-format off
1342 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1343 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1344 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1345
1346 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1347 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1348 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1349 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1350 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1351 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1352
1353 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1354 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1355 case ISD::STRICT_FMINNUM:
1356 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1357 case ISD::STRICT_FMAXNUM:
1358 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1359 case ISD::STRICT_FADD:
1360 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1361 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1362 case ISD::STRICT_FCEIL:
1363 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1364 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1365 case ISD::STRICT_FCOS:
1366 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1367 case ISD::STRICT_FDIV:
1368 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1369 case ISD::STRICT_FEXP:
1370 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1371 case ISD::STRICT_FEXP2:
1372 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1373 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1374 case ISD::STRICT_FFLOOR:
1375 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1376 case ISD::STRICT_FLOG:
1377 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1378 case ISD::STRICT_FLOG2:
1379 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1380 case ISD::STRICT_FLOG10:
1381 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1382 case ISD::STRICT_FMA:
1383 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1384 case ISD::STRICT_FMUL:
1385 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1386 case ISD::STRICT_FNEARBYINT:
1387 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1388 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1389 case ISD::STRICT_FP_EXTEND:
1390 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1391 case ISD::STRICT_FPOW:
1392 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1393 case ISD::STRICT_FPOWI:
1394 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1395 case ISD::FLDEXP:
1396 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1397 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1398 case ISD::STRICT_FRINT:
1399 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1400 case ISD::STRICT_FROUND:
1401 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1402 case ISD::STRICT_FROUNDEVEN:
1403 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1404 case ISD::STRICT_FSIN:
1405 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1406 case ISD::STRICT_FSQRT:
1407 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1408 case ISD::STRICT_FSUB:
1409 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1410 case ISD::STRICT_FTAN:
1411 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1412 case ISD::STRICT_FTRUNC:
1413 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1414 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1415 case ISD::STRICT_SINT_TO_FP:
1416 case ISD::STRICT_UINT_TO_FP:
1417 case ISD::SINT_TO_FP:
1418 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1419 case ISD::STRICT_FREM:
1420 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1421 // clang-format on
1422 }
1423
1424 // If Lo/Hi is null, the sub-method took care of registering results etc.
1425 if (Lo.getNode())
1426 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1427}
1428
1429void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1430 SDValue &Hi) {
1431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1432 assert(NVT.getSizeInBits() == 64 &&
1433 "Do not know how to expand this float constant!");
1434 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1435 SDLoc dl(N);
1436 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1437 APInt(64, C.getRawData()[1])),
1438 dl, NVT);
1439 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1440 APInt(64, C.getRawData()[0])),
1441 dl, NVT);
1442}
1443
1444void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1445 SDValue &Lo, SDValue &Hi) {
1446 bool IsStrict = N->isStrictFPOpcode();
1447 unsigned Offset = IsStrict ? 1 : 0;
1448 SDValue Op = N->getOperand(0 + Offset);
1449 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1450 TargetLowering::MakeLibCallOptions CallOptions;
1451 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1452 Op, CallOptions, SDLoc(N),
1453 Chain);
1454 if (IsStrict)
1455 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1456 GetPairElements(Tmp.first, Lo, Hi);
1457}
1458
1459void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1460 SDValue &Lo, SDValue &Hi) {
1461 bool IsStrict = N->isStrictFPOpcode();
1462 unsigned Offset = IsStrict ? 1 : 0;
1463 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1464 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1465 TargetLowering::MakeLibCallOptions CallOptions;
1466 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1467 Ops, CallOptions, SDLoc(N),
1468 Chain);
1469 if (IsStrict)
1470 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1471 GetPairElements(Tmp.first, Lo, Hi);
1472}
1473
1474void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1475 SDValue &Hi) {
1476 assert(N->getValueType(0) == MVT::ppcf128 &&
1477 "Logic only correct for ppcf128!");
1478 SDLoc dl(N);
1479 SDValue Tmp;
1480 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1481 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1482 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1483 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1484 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1485 ISD::SETEQ);
1486}
1487
1488void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1489 SDValue &Hi) {
1490 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1491 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1492 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1493 RTLIB::FMIN_PPCF128), Lo, Hi);
1494}
1495
1496void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1497 SDValue &Hi) {
1498 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1499 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1500 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1501 RTLIB::FMAX_PPCF128), Lo, Hi);
1502}
1503
1504void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1505 SDValue &Hi) {
1506 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1507 RTLIB::ADD_F32, RTLIB::ADD_F64,
1508 RTLIB::ADD_F80, RTLIB::ADD_F128,
1509 RTLIB::ADD_PPCF128), Lo, Hi);
1510}
1511
1512void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1513 SDValue &Hi) {
1514 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1515 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1516 RTLIB::CBRT_F128,
1517 RTLIB::CBRT_PPCF128), Lo, Hi);
1518}
1519
1520void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1521 SDValue &Lo, SDValue &Hi) {
1522 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1523 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1524 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1525 RTLIB::CEIL_PPCF128), Lo, Hi);
1526}
1527
1528void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1529 SDValue &Lo, SDValue &Hi) {
1530 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1531 RTLIB::COPYSIGN_F32,
1532 RTLIB::COPYSIGN_F64,
1533 RTLIB::COPYSIGN_F80,
1534 RTLIB::COPYSIGN_F128,
1535 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1536}
1537
1538void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1539 SDValue &Lo, SDValue &Hi) {
1540 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1541 RTLIB::COS_F32, RTLIB::COS_F64,
1542 RTLIB::COS_F80, RTLIB::COS_F128,
1543 RTLIB::COS_PPCF128), Lo, Hi);
1544}
1545
1546void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1547 SDValue &Hi) {
1548 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1549 RTLIB::DIV_F32,
1550 RTLIB::DIV_F64,
1551 RTLIB::DIV_F80,
1552 RTLIB::DIV_F128,
1553 RTLIB::DIV_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::EXP_F32, RTLIB::EXP_F64,
1560 RTLIB::EXP_F80, RTLIB::EXP_F128,
1561 RTLIB::EXP_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1565 SDValue &Lo, SDValue &Hi) {
1566 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1567 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1568 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1569 RTLIB::EXP2_PPCF128), Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Unary(N,
1575 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1576 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1577 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1578 Lo, Hi);
1579}
1580
1581void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1582 SDValue &Lo, SDValue &Hi) {
1583 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1584 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1585 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1586 RTLIB::FLOOR_PPCF128), Lo, Hi);
1587}
1588
1589void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1590 SDValue &Lo, SDValue &Hi) {
1591 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1592 RTLIB::LOG_F32, RTLIB::LOG_F64,
1593 RTLIB::LOG_F80, RTLIB::LOG_F128,
1594 RTLIB::LOG_PPCF128), Lo, Hi);
1595}
1596
1597void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1598 SDValue &Lo, SDValue &Hi) {
1599 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1600 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1601 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1602 RTLIB::LOG2_PPCF128), Lo, Hi);
1603}
1604
1605void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1606 SDValue &Lo, SDValue &Hi) {
1607 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1608 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1609 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1610 RTLIB::LOG10_PPCF128), Lo, Hi);
1611}
1612
1613void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1614 SDValue &Hi) {
1615 bool IsStrict = N->isStrictFPOpcode();
1616 unsigned Offset = IsStrict ? 1 : 0;
1617 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1618 N->getOperand(2 + Offset) };
1619 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1620 TargetLowering::MakeLibCallOptions CallOptions;
1621 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1622 RTLIB::FMA_F32,
1623 RTLIB::FMA_F64,
1624 RTLIB::FMA_F80,
1625 RTLIB::FMA_F128,
1626 RTLIB::FMA_PPCF128),
1627 N->getValueType(0), Ops, CallOptions,
1628 SDLoc(N), Chain);
1629 if (IsStrict)
1630 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1631 GetPairElements(Tmp.first, Lo, Hi);
1632}
1633
1634void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1635 SDValue &Hi) {
1636 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1637 RTLIB::MUL_F32,
1638 RTLIB::MUL_F64,
1639 RTLIB::MUL_F80,
1640 RTLIB::MUL_F128,
1641 RTLIB::MUL_PPCF128), Lo, Hi);
1642}
1643
1644void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1645 SDValue &Lo, SDValue &Hi) {
1646 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1647 RTLIB::NEARBYINT_F32,
1648 RTLIB::NEARBYINT_F64,
1649 RTLIB::NEARBYINT_F80,
1650 RTLIB::NEARBYINT_F128,
1651 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1652}
1653
1654void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1655 SDValue &Hi) {
1656 SDLoc dl(N);
1657 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1658 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1659 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1660}
1661
1662void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1663 SDValue &Hi) {
1664 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1665 SDLoc dl(N);
1666 bool IsStrict = N->isStrictFPOpcode();
1667
1668 SDValue Chain;
1669 if (IsStrict) {
1670 // If the expanded type is the same as the input type, just bypass the node.
1671 if (NVT == N->getOperand(1).getValueType()) {
1672 Hi = N->getOperand(1);
1673 Chain = N->getOperand(0);
1674 } else {
1675 // Other we need to extend.
1676 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1677 { N->getOperand(0), N->getOperand(1) });
1678 Chain = Hi.getValue(1);
1679 }
1680 } else {
1681 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1682 }
1683
1684 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1685 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1686
1687 if (IsStrict)
1688 ReplaceValueWith(SDValue(N, 1), Chain);
1689}
1690
1691void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1692 SDValue &Lo, SDValue &Hi) {
1693 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1694 RTLIB::POW_F32, RTLIB::POW_F64,
1695 RTLIB::POW_F80, RTLIB::POW_F128,
1696 RTLIB::POW_PPCF128), Lo, Hi);
1697}
1698
1699void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1700 SDValue &Lo, SDValue &Hi) {
1701 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1702}
1703
1704void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1705 SDValue &Hi) {
1706 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1707}
1708
1709void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1710 SDValue &Lo, SDValue &Hi) {
1711 assert(N->getValueType(0) == MVT::ppcf128 &&
1712 "Logic only correct for ppcf128!");
1713
1714 SDLoc dl(N);
1715 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1716 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1717 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1721 SDValue &Lo, SDValue &Hi) {
1722 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1723 RTLIB::REM_F32, RTLIB::REM_F64,
1724 RTLIB::REM_F80, RTLIB::REM_F128,
1725 RTLIB::REM_PPCF128), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1729 SDValue &Lo, SDValue &Hi) {
1730 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1731 RTLIB::RINT_F32, RTLIB::RINT_F64,
1732 RTLIB::RINT_F80, RTLIB::RINT_F128,
1733 RTLIB::RINT_PPCF128), Lo, Hi);
1734}
1735
1736void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1737 SDValue &Lo, SDValue &Hi) {
1738 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1739 RTLIB::ROUND_F32,
1740 RTLIB::ROUND_F64,
1741 RTLIB::ROUND_F80,
1742 RTLIB::ROUND_F128,
1743 RTLIB::ROUND_PPCF128), Lo, Hi);
1744}
1745
1746void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1747 SDValue &Lo, SDValue &Hi) {
1748 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1749 RTLIB::ROUNDEVEN_F32,
1750 RTLIB::ROUNDEVEN_F64,
1751 RTLIB::ROUNDEVEN_F80,
1752 RTLIB::ROUNDEVEN_F128,
1753 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1754}
1755
1756void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1757 SDValue &Lo, SDValue &Hi) {
1758 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1759 RTLIB::SIN_F32, RTLIB::SIN_F64,
1760 RTLIB::SIN_F80, RTLIB::SIN_F128,
1761 RTLIB::SIN_PPCF128), Lo, Hi);
1762}
1763
1764void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1765 SDValue &Lo, SDValue &Hi) {
1766 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1767 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1768 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1769 RTLIB::SQRT_PPCF128), Lo, Hi);
1770}
1771
1772void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1773 SDValue &Hi) {
1774 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1775 RTLIB::SUB_F32,
1776 RTLIB::SUB_F64,
1777 RTLIB::SUB_F80,
1778 RTLIB::SUB_F128,
1779 RTLIB::SUB_PPCF128), Lo, Hi);
1780}
1781
1782void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1783 SDValue &Hi) {
1784 ExpandFloatRes_Unary(N,
1785 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
1786 RTLIB::TAN_F64, RTLIB::TAN_F80,
1787 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
1788 Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1792 SDValue &Lo, SDValue &Hi) {
1793 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1794 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1795 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1796 RTLIB::TRUNC_PPCF128), Lo, Hi);
1797}
1798
1799void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1800 SDValue &Hi) {
1801 if (ISD::isNormalLoad(N)) {
1802 ExpandRes_NormalLoad(N, Lo, Hi);
1803 return;
1804 }
1805
1806 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1807 LoadSDNode *LD = cast<LoadSDNode>(N);
1808 SDValue Chain = LD->getChain();
1809 SDValue Ptr = LD->getBasePtr();
1810 SDLoc dl(N);
1811
1812 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1813 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1814 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1815
1816 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1817 LD->getMemoryVT(), LD->getMemOperand());
1818
1819 // Remember the chain.
1820 Chain = Hi.getValue(1);
1821
1822 // The low part is zero.
1823 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1824 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1825
1826 // Modified the chain - switch anything that used the old chain to use the
1827 // new one.
1828 ReplaceValueWith(SDValue(LD, 1), Chain);
1829}
1830
1831void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1832 SDValue &Hi) {
1833 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1834 EVT VT = N->getValueType(0);
1835 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1836 bool Strict = N->isStrictFPOpcode();
1837 SDValue Src = N->getOperand(Strict ? 1 : 0);
1838 EVT SrcVT = Src.getValueType();
1839 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1840 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1841 SDLoc dl(N);
1842 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1843
1844 // TODO: Any other flags to propagate?
1845 SDNodeFlags Flags;
1846 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1847
1848 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1849 // When promoting partial word types to i32 we must honor the signedness,
1850 // though.
1851 if (SrcVT.bitsLE(MVT::i32)) {
1852 // The integer can be represented exactly in an f64.
1853 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1854 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1855 if (Strict) {
1856 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1857 {Chain, Src}, Flags);
1858 Chain = Hi.getValue(1);
1859 } else
1860 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1861 } else {
1862 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1863 if (SrcVT.bitsLE(MVT::i64)) {
1864 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1865 MVT::i64, Src);
1866 LC = RTLIB::SINTTOFP_I64_PPCF128;
1867 } else if (SrcVT.bitsLE(MVT::i128)) {
1868 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1869 LC = RTLIB::SINTTOFP_I128_PPCF128;
1870 }
1871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1872
1873 TargetLowering::MakeLibCallOptions CallOptions;
1874 CallOptions.setSExt(true);
1875 std::pair<SDValue, SDValue> Tmp =
1876 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1877 if (Strict)
1878 Chain = Tmp.second;
1879 GetPairElements(Tmp.first, Lo, Hi);
1880 }
1881
1882 // No need to complement for unsigned 32-bit integers
1883 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1884 if (Strict)
1885 ReplaceValueWith(SDValue(N, 1), Chain);
1886
1887 return;
1888 }
1889
1890 // Unsigned - fix up the SINT_TO_FP value just calculated.
1891 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1892 // keep semantics correctness if the integer is not exactly representable
1893 // here. See ExpandLegalINT_TO_FP.
1894 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1895 SrcVT = Src.getValueType();
1896
1897 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1898 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1899 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1900 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1901 ArrayRef<uint64_t> Parts;
1902
1903 switch (SrcVT.getSimpleVT().SimpleTy) {
1904 default:
1905 llvm_unreachable("Unsupported UINT_TO_FP!");
1906 case MVT::i32:
1907 Parts = TwoE32;
1908 break;
1909 case MVT::i64:
1910 Parts = TwoE64;
1911 break;
1912 case MVT::i128:
1913 Parts = TwoE128;
1914 break;
1915 }
1916
1917 // TODO: Are there other fast-math-flags to propagate to this FADD?
1918 SDValue NewLo = DAG.getConstantFP(
1919 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1920 if (Strict) {
1921 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1922 {Chain, Hi, NewLo}, Flags);
1923 Chain = Lo.getValue(1);
1924 ReplaceValueWith(SDValue(N, 1), Chain);
1925 } else
1926 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1927 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1928 Lo, Hi, ISD::SETLT);
1929 GetPairElements(Lo, Lo, Hi);
1930}
1931
1932
1933//===----------------------------------------------------------------------===//
1934// Float Operand Expansion
1935//===----------------------------------------------------------------------===//
1936
1937/// ExpandFloatOperand - This method is called when the specified operand of the
1938/// specified node is found to need expansion. At this point, all of the result
1939/// types of the node are known to be legal, but other operands of the node may
1940/// need promotion or expansion as well as the specified one.
1941bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1942 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1943 SDValue Res = SDValue();
1944
1945 // See if the target wants to custom expand this node.
1946 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1947 return false;
1948
1949 switch (N->getOpcode()) {
1950 default:
1951#ifndef NDEBUG
1952 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1953 N->dump(&DAG); dbgs() << "\n";
1954#endif
1955 report_fatal_error("Do not know how to expand this operator's operand!");
1956
1957 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1958 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1959 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1960
1961 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1962 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1963 case ISD::STRICT_FP_ROUND:
1964 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1965 case ISD::STRICT_FP_TO_SINT:
1966 case ISD::STRICT_FP_TO_UINT:
1967 case ISD::FP_TO_SINT:
1968 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1969 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1970 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1971 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1972 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1973 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1974 case ISD::STRICT_FSETCC:
1975 case ISD::STRICT_FSETCCS:
1976 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1977 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1978 OpNo); break;
1979 }
1980
1981 // If the result is null, the sub-method took care of registering results etc.
1982 if (!Res.getNode()) return false;
1983
1984 // If the result is N, the sub-method updated N in place. Tell the legalizer
1985 // core about this.
1986 if (Res.getNode() == N)
1987 return true;
1988
1989 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1990 "Invalid operand expansion");
1991
1992 ReplaceValueWith(SDValue(N, 0), Res);
1993 return false;
1994}
1995
1996/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1997/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1998void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1999 SDValue &NewRHS,
2000 ISD::CondCode &CCCode,
2001 const SDLoc &dl, SDValue &Chain,
2002 bool IsSignaling) {
2003 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2004 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2005 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2006
2007 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2008
2009 // FIXME: This generated code sucks. We want to generate
2010 // FCMPU crN, hi1, hi2
2011 // BNE crN, L:
2012 // FCMPU crN, lo1, lo2
2013 // The following can be improved, but not that much.
2014 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2015 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2016 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2017 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2018 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2019 RHSLo, CCCode, OutputChain, IsSignaling);
2020 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2021 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2022 Tmp1 =
2023 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2024 ISD::SETUNE, OutputChain, IsSignaling);
2025 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2026 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2027 RHSHi, CCCode, OutputChain, IsSignaling);
2028 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2029 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2030 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2031 NewRHS = SDValue(); // LHS is the result, not a compare.
2032 Chain = OutputChain;
2033}
2034
2035SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2036 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2037 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2038 SDValue Chain;
2039 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2040
2041 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2042 // against zero to select between true and false values.
2043 if (!NewRHS.getNode()) {
2044 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2045 CCCode = ISD::SETNE;
2046 }
2047
2048 // Update N to have the operands specified.
2049 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2050 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2051 N->getOperand(4)), 0);
2052}
2053
2054SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2055 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2056 "Logic only correct for ppcf128!");
2057 SDValue Lo, Hi;
2058 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2059 // The ppcf128 value is providing only the sign; take it from the
2060 // higher-order double (which must have the larger magnitude).
2061 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2062 N->getValueType(0), N->getOperand(0), Hi);
2063}
2064
2065SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2066 bool IsStrict = N->isStrictFPOpcode();
2067 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2068 "Logic only correct for ppcf128!");
2069 SDValue Lo, Hi;
2070 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2071
2072 if (!IsStrict)
2073 // Round it the rest of the way (e.g. to f32) if needed.
2074 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2075 N->getValueType(0), Hi, N->getOperand(1));
2076
2077 // Eliminate the node if the input float type is the same as the output float
2078 // type.
2079 if (Hi.getValueType() == N->getValueType(0)) {
2080 // Connect the output chain to the input chain, unlinking the node.
2081 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2082 ReplaceValueWith(SDValue(N, 0), Hi);
2083 return SDValue();
2084 }
2085
2086 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2087 {N->getValueType(0), MVT::Other},
2088 {N->getOperand(0), Hi, N->getOperand(2)});
2089 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2090 ReplaceValueWith(SDValue(N, 0), Expansion);
2091 return SDValue();
2092}
2093
2094SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2095 EVT RVT = N->getValueType(0);
2096 SDLoc dl(N);
2097
2098 bool IsStrict = N->isStrictFPOpcode();
2099 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2100 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2101 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2102 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2103
2104 EVT NVT;
2105 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2106 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2107 "Unsupported FP_TO_XINT!");
2108 TargetLowering::MakeLibCallOptions CallOptions;
2109 std::pair<SDValue, SDValue> Tmp =
2110 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2111 if (!IsStrict)
2112 return Tmp.first;
2113
2114 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2115 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2116 return SDValue();
2117}
2118
2119SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2120 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2121 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2122 SDValue Chain;
2123 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2124
2125 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2126 // against zero to select between true and false values.
2127 if (!NewRHS.getNode()) {
2128 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2129 CCCode = ISD::SETNE;
2130 }
2131
2132 // Update N to have the operands specified.
2133 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2134 N->getOperand(2), N->getOperand(3),
2135 DAG.getCondCode(CCCode)), 0);
2136}
2137
2138SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2139 bool IsStrict = N->isStrictFPOpcode();
2140 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2141 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2142 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2143 ISD::CondCode CCCode =
2144 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2145 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2146 N->getOpcode() == ISD::STRICT_FSETCCS);
2147
2148 // FloatExpandSetCCOperands always returned a scalar.
2149 assert(!NewRHS.getNode() && "Expect to return scalar");
2150 assert(NewLHS.getValueType() == N->getValueType(0) &&
2151 "Unexpected setcc expansion!");
2152 if (Chain) {
2153 ReplaceValueWith(SDValue(N, 0), NewLHS);
2154 ReplaceValueWith(SDValue(N, 1), Chain);
2155 return SDValue();
2156 }
2157 return NewLHS;
2158}
2159
2160SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2161 if (ISD::isNormalStore(N))
2162 return ExpandOp_NormalStore(N, OpNo);
2163
2164 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2165 assert(OpNo == 1 && "Can only expand the stored value so far");
2166 StoreSDNode *ST = cast<StoreSDNode>(N);
2167
2168 SDValue Chain = ST->getChain();
2169 SDValue Ptr = ST->getBasePtr();
2170
2171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2172 ST->getValue().getValueType());
2173 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2174 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2175 (void)NVT;
2176
2177 SDValue Lo, Hi;
2178 GetExpandedOp(ST->getValue(), Lo, Hi);
2179
2180 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2181 ST->getMemoryVT(), ST->getMemOperand());
2182}
2183
2184SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2185 EVT RVT = N->getValueType(0);
2186 EVT RetVT = N->getOperand(0).getValueType();
2187 TargetLowering::MakeLibCallOptions CallOptions;
2188 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2189 RTLIB::LROUND_F32,
2190 RTLIB::LROUND_F64,
2191 RTLIB::LROUND_F80,
2192 RTLIB::LROUND_F128,
2193 RTLIB::LROUND_PPCF128),
2194 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2195}
2196
2197SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2198 EVT RVT = N->getValueType(0);
2199 EVT RetVT = N->getOperand(0).getValueType();
2200 TargetLowering::MakeLibCallOptions CallOptions;
2201 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2202 RTLIB::LLROUND_F32,
2203 RTLIB::LLROUND_F64,
2204 RTLIB::LLROUND_F80,
2205 RTLIB::LLROUND_F128,
2206 RTLIB::LLROUND_PPCF128),
2207 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2208}
2209
2210SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2211 EVT RVT = N->getValueType(0);
2212 EVT RetVT = N->getOperand(0).getValueType();
2213 TargetLowering::MakeLibCallOptions CallOptions;
2214 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2215 RTLIB::LRINT_F32,
2216 RTLIB::LRINT_F64,
2217 RTLIB::LRINT_F80,
2218 RTLIB::LRINT_F128,
2219 RTLIB::LRINT_PPCF128),
2220 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2221}
2222
2223SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2224 EVT RVT = N->getValueType(0);
2225 EVT RetVT = N->getOperand(0).getValueType();
2226 TargetLowering::MakeLibCallOptions CallOptions;
2227 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2228 RTLIB::LLRINT_F32,
2229 RTLIB::LLRINT_F64,
2230 RTLIB::LLRINT_F80,
2231 RTLIB::LLRINT_F128,
2232 RTLIB::LLRINT_PPCF128),
2233 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2234}
2235
2236//===----------------------------------------------------------------------===//
2237// Float Operand Promotion
2238//===----------------------------------------------------------------------===//
2239//
2240
2241static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2242 if (OpVT == MVT::f16) {
2243 return ISD::FP16_TO_FP;
2244 } else if (RetVT == MVT::f16) {
2245 return ISD::FP_TO_FP16;
2246 } else if (OpVT == MVT::bf16) {
2247 return ISD::BF16_TO_FP;
2248 } else if (RetVT == MVT::bf16) {
2249 return ISD::FP_TO_BF16;
2250 }
2251
2252 report_fatal_error("Attempt at an invalid promotion-related conversion");
2253}
2254
2255static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2256 if (OpVT == MVT::f16)
2257 return ISD::STRICT_FP16_TO_FP;
2258
2259 if (RetVT == MVT::f16)
2260 return ISD::STRICT_FP_TO_FP16;
2261
2262 if (OpVT == MVT::bf16)
2263 return ISD::STRICT_BF16_TO_FP;
2264
2265 if (RetVT == MVT::bf16)
2266 return ISD::STRICT_FP_TO_BF16;
2267
2268 report_fatal_error("Attempt at an invalid promotion-related conversion");
2269}
2270
2271bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2272 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2273 SDValue R = SDValue();
2274
2275 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2276 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2277 return false;
2278 }
2279
2280 // Nodes that use a promotion-requiring floating point operand, but doesn't
2281 // produce a promotion-requiring floating point result, need to be legalized
2282 // to use the promoted float operand. Nodes that produce at least one
2283 // promotion-requiring floating point result have their operands legalized as
2284 // a part of PromoteFloatResult.
2285 // clang-format off
2286 switch (N->getOpcode()) {
2287 default:
2288 #ifndef NDEBUG
2289 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2290 N->dump(&DAG); dbgs() << "\n";
2291 #endif
2292 report_fatal_error("Do not know how to promote this operator's operand!");
2293
2294 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2295 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2296 case ISD::FP_TO_SINT:
2297 case ISD::FP_TO_UINT:
2298 case ISD::LRINT:
2299 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2300 case ISD::FP_TO_SINT_SAT:
2301 case ISD::FP_TO_UINT_SAT:
2302 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2303 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2304 case ISD::STRICT_FP_EXTEND:
2305 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2306 break;
2307 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2308 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2309 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2310 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2311 }
2312 // clang-format on
2313
2314 if (R.getNode())
2315 ReplaceValueWith(SDValue(N, 0), R);
2316 return false;
2317}
2318
2319SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2320 SDValue Op = N->getOperand(0);
2321 EVT OpVT = Op->getValueType(0);
2322
2323 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2324 EVT PromotedVT = Promoted->getValueType(0);
2325
2326 // Convert the promoted float value to the desired IVT.
2327 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2328 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2329 IVT, Promoted);
2330 // The final result type might not be an scalar so we need a bitcast. The
2331 // bitcast will be further legalized if needed.
2332 return DAG.getBitcast(N->getValueType(0), Convert);
2333}
2334
2335// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2336// PromoteFloatRes_FCOPYSIGN.
2337SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2338 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2339 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2340
2341 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2342 N->getOperand(0), Op1);
2343}
2344
2345// Convert the promoted float value to the desired integer type
2346SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2347 SDValue Op = GetPromotedFloat(N->getOperand(0));
2348 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2349}
2350
2351SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2352 unsigned OpNo) {
2353 SDValue Op = GetPromotedFloat(N->getOperand(0));
2354 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2355 N->getOperand(1));
2356}
2357
2358SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2359 SDValue Op = GetPromotedFloat(N->getOperand(0));
2360 EVT VT = N->getValueType(0);
2361
2362 // Desired VT is same as promoted type. Use promoted float directly.
2363 if (VT == Op->getValueType(0))
2364 return Op;
2365
2366 // Else, extend the promoted float value to the desired VT.
2367 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2368}
2369
2370SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2371 unsigned OpNo) {
2372 assert(OpNo == 1 && "Promoting unpromotable operand");
2373
2374 SDValue Op = GetPromotedFloat(N->getOperand(1));
2375 EVT VT = N->getValueType(0);
2376
2377 // Desired VT is same as promoted type. Use promoted float directly.
2378 if (VT == Op->getValueType(0)) {
2379 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2380 return Op;
2381 }
2382
2383 // Else, extend the promoted float value to the desired VT.
2384 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2385 N->getOperand(0), Op);
2386 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2387 return Res;
2388}
2389
2390// Promote the float operands used for comparison. The true- and false-
2391// operands have the same type as the result and are promoted, if needed, by
2392// PromoteFloatRes_SELECT_CC
2393SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2394 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2395 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2396
2397 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2398 LHS, RHS, N->getOperand(2), N->getOperand(3),
2399 N->getOperand(4));
2400}
2401
2402// Construct a SETCC that compares the promoted values and sets the conditional
2403// code.
2404SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2405 EVT VT = N->getValueType(0);
2406 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2407 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2408 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2409
2410 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2411
2412}
2413
2414// Lower the promoted Float down to the integer value of same size and construct
2415// a STORE of the integer value.
2416SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2417 StoreSDNode *ST = cast<StoreSDNode>(N);
2418 SDValue Val = ST->getValue();
2419 SDLoc DL(N);
2420
2421 SDValue Promoted = GetPromotedFloat(Val);
2422 EVT VT = ST->getOperand(1).getValueType();
2423 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2424
2425 SDValue NewVal;
2426 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2427 IVT, Promoted);
2428
2429 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2430 ST->getMemOperand());
2431}
2432
2433SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2434 unsigned OpNo) {
2435 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2436 SDValue Val = ST->getVal();
2437 SDLoc DL(N);
2438
2439 SDValue Promoted = GetPromotedFloat(Val);
2440 EVT VT = ST->getOperand(1).getValueType();
2441 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2442
2443 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2444 DL, IVT, Promoted);
2445
2446 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2447 ST->getBasePtr(), ST->getMemOperand());
2448}
2449
2450//===----------------------------------------------------------------------===//
2451// Float Result Promotion
2452//===----------------------------------------------------------------------===//
2453
2454void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2455 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2456 SDValue R = SDValue();
2457
2458 // See if the target wants to custom expand this node.
2459 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2460 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2461 return;
2462 }
2463
2464 switch (N->getOpcode()) {
2465 // These opcodes cannot appear if promotion of FP16 is done in the backend
2466 // instead of Clang
2467 case ISD::FP16_TO_FP:
2468 case ISD::FP_TO_FP16:
2469 default:
2470#ifndef NDEBUG
2471 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2472 N->dump(&DAG); dbgs() << "\n";
2473#endif
2474 report_fatal_error("Do not know how to promote this operator's result!");
2475
2476 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2477 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2478 case ISD::EXTRACT_VECTOR_ELT:
2479 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2480 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2481
2482 // Unary FP Operations
2483 case ISD::FABS:
2484 case ISD::FCBRT:
2485 case ISD::FCEIL:
2486 case ISD::FCOS:
2487 case ISD::FEXP:
2488 case ISD::FEXP2:
2489 case ISD::FEXP10:
2490 case ISD::FFLOOR:
2491 case ISD::FLOG:
2492 case ISD::FLOG2:
2493 case ISD::FLOG10:
2494 case ISD::FNEARBYINT:
2495 case ISD::FNEG:
2496 case ISD::FRINT:
2497 case ISD::FROUND:
2498 case ISD::FROUNDEVEN:
2499 case ISD::FSIN:
2500 case ISD::FSQRT:
2501 case ISD::FTRUNC:
2502 case ISD::FTAN:
2503 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2504
2505 // Binary FP Operations
2506 case ISD::FADD:
2507 case ISD::FDIV:
2508 case ISD::FMAXIMUM:
2509 case ISD::FMINIMUM:
2510 case ISD::FMAXNUM:
2511 case ISD::FMINNUM:
2512 case ISD::FMAXNUM_IEEE:
2513 case ISD::FMINNUM_IEEE:
2514 case ISD::FMUL:
2515 case ISD::FPOW:
2516 case ISD::FREM:
2517 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2518
2519 case ISD::FMA: // FMA is same as FMAD
2520 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2521
2522 case ISD::FPOWI:
2523 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2524 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2525
2526 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2527 case ISD::STRICT_FP_ROUND:
2528 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2529 break;
2530 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2531 case ISD::ATOMIC_LOAD:
2532 R = PromoteFloatRes_ATOMIC_LOAD(N);
2533 break;
2534 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2535 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2536
2537 case ISD::SINT_TO_FP:
2538 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2539 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2540 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2541 case ISD::VECREDUCE_FADD:
2542 case ISD::VECREDUCE_FMUL:
2543 case ISD::VECREDUCE_FMIN:
2544 case ISD::VECREDUCE_FMAX:
2545 case ISD::VECREDUCE_FMAXIMUM:
2546 case ISD::VECREDUCE_FMINIMUM:
2547 R = PromoteFloatRes_VECREDUCE(N);
2548 break;
2549 case ISD::VECREDUCE_SEQ_FADD:
2550 case ISD::VECREDUCE_SEQ_FMUL:
2551 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2552 break;
2553 }
2554
2555 if (R.getNode())
2556 SetPromotedFloat(SDValue(N, ResNo), R);
2557}
2558
2559// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2560// At this point, it is not possible to determine if the bitcast value is
2561// eventually stored to memory or promoted to f32 or promoted to a floating
2562// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2563// STORE promotion handlers.
2564SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2565 EVT VT = N->getValueType(0);
2566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2567 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2568 // bitcast will be legalized further if necessary.
2569 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2570 N->getOperand(0).getValueType().getSizeInBits());
2571 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2572 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2573}
2574
2575SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2576 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2577 EVT VT = N->getValueType(0);
2578 SDLoc DL(N);
2579
2580 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2581 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2582 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2583 IVT);
2584
2585 // Convert the Constant to the desired FP type
2586 // FIXME We might be able to do the conversion during compilation and get rid
2587 // of it from the object code
2588 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2589 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2590}
2591
2592// If the Index operand is a constant, try to redirect the extract operation to
2593// the correct legalized vector. If not, bit-convert the input vector to
2594// equivalent integer vector. Extract the element as an (bit-cast) integer
2595// value and convert it to the promoted type.
2596SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2597 SDLoc DL(N);
2598
2599 // If the index is constant, try to extract the value from the legalized
2600 // vector type.
2601 if (isa<ConstantSDNode>(N->getOperand(1))) {
2602 SDValue Vec = N->getOperand(0);
2603 SDValue Idx = N->getOperand(1);
2604 EVT VecVT = Vec->getValueType(0);
2605 EVT EltVT = VecVT.getVectorElementType();
2606
2607 uint64_t IdxVal = Idx->getAsZExtVal();
2608
2609 switch (getTypeAction(VecVT)) {
2610 default: break;
2611 case TargetLowering::TypeScalarizeVector: {
2612 SDValue Res = GetScalarizedVector(N->getOperand(0));
2613 ReplaceValueWith(SDValue(N, 0), Res);
2614 return SDValue();
2615 }
2616 case TargetLowering::TypeWidenVector: {
2617 Vec = GetWidenedVector(Vec);
2618 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2619 ReplaceValueWith(SDValue(N, 0), Res);
2620 return SDValue();
2621 }
2622 case TargetLowering::TypeSplitVector: {
2623 SDValue Lo, Hi;
2624 GetSplitVector(Vec, Lo, Hi);
2625
2626 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2627 SDValue Res;
2628 if (IdxVal < LoElts)
2629 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2630 else
2631 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2632 DAG.getConstant(IdxVal - LoElts, DL,
2633 Idx.getValueType()));
2634 ReplaceValueWith(SDValue(N, 0), Res);
2635 return SDValue();
2636 }
2637
2638 }
2639 }
2640
2641 // Bit-convert the input vector to the equivalent integer vector
2642 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2643 EVT IVT = NewOp.getValueType().getVectorElementType();
2644
2645 // Extract the element as an (bit-cast) integer value
2646 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2647 NewOp, N->getOperand(1));
2648
2649 // Convert the element to the desired FP type
2650 EVT VT = N->getValueType(0);
2651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2652 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2653}
2654
2655// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2656// needs promotion, so does the argument X. Note that Y, if needed, will be
2657// handled during operand promotion.
2658SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2659 EVT VT = N->getValueType(0);
2660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2661 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2662
2663 SDValue Op1 = N->getOperand(1);
2664
2665 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2666}
2667
2668// Unary operation where the result and the operand have PromoteFloat type
2669// action. Construct a new SDNode with the promoted float value of the old
2670// operand.
2671SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2672 EVT VT = N->getValueType(0);
2673 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2674 SDValue Op = GetPromotedFloat(N->getOperand(0));
2675
2676 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2677}
2678
2679// Binary operations where the result and both operands have PromoteFloat type
2680// action. Construct a new SDNode with the promoted float values of the old
2681// operands.
2682SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2683 EVT VT = N->getValueType(0);
2684 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2685 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2686 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2687 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2688}
2689
2690SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2691 EVT VT = N->getValueType(0);
2692 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2693 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2694 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2695 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2696
2697 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2698}
2699
2700// Promote the Float (first) operand and retain the Integer (second) operand
2701SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2702 EVT VT = N->getValueType(0);
2703 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2704 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2705 SDValue Op1 = N->getOperand(1);
2706
2707 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2708}
2709
2710SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2711 EVT VT = N->getValueType(0);
2712 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2713 SDValue Op = GetPromotedFloat(N->getOperand(0));
2714 SDValue Res =
2715 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2716
2717 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2718 return Res;
2719}
2720
2721// Explicit operation to reduce precision. Reduce the value to half precision
2722// and promote it back to the legal type.
2723SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2724 SDLoc DL(N);
2725
2726 SDValue Op = N->getOperand(0);
2727 EVT VT = N->getValueType(0);
2728 EVT OpVT = Op->getValueType(0);
2729 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2730 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2731
2732 // Round promoted float to desired precision
2733 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2734 // Promote it back to the legal output type
2735 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2736}
2737
2738// Explicit operation to reduce precision. Reduce the value to half precision
2739// and promote it back to the legal type.
2740SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2741 SDLoc DL(N);
2742
2743 SDValue Chain = N->getOperand(0);
2744 SDValue Op = N->getOperand(1);
2745 EVT VT = N->getValueType(0);
2746 EVT OpVT = Op->getValueType(0);
2747 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2748 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2749
2750 // Round promoted float to desired precision
2751 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2752 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2753 // Promote it back to the legal output type
2754 SDValue Res =
2755 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2756 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2757 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2758 return Res;
2759}
2760
2761SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2762 LoadSDNode *L = cast<LoadSDNode>(N);
2763 EVT VT = N->getValueType(0);
2764
2765 // Load the value as an integer value with the same number of bits.
2766 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2767 SDValue newL = DAG.getLoad(
2768 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2769 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2770 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2771 // Legalize the chain result by replacing uses of the old value chain with the
2772 // new one
2773 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2774
2775 // Convert the integer value to the desired FP type
2776 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2777 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2778}
2779
2780SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2781 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2782 EVT VT = AM->getValueType(0);
2783
2784 // Load the value as an integer value with the same number of bits.
2785 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2786 SDValue newL = DAG.getAtomic(
2787 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
2788 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
2789
2790 // Legalize the chain result by replacing uses of the old value chain with the
2791 // new one
2792 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2793
2794 // Convert the integer value to the desired FP type
2795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2796 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
2797}
2798
2799// Construct a new SELECT node with the promoted true- and false- values.
2800SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2801 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2802 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2803
2804 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2805 N->getOperand(0), TrueVal, FalseVal);
2806}
2807
2808// Construct a new SELECT_CC node with the promoted true- and false- values.
2809// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2810SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2811 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2812 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2813
2814 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2815 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2816 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2817}
2818
2819// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2820// float type.
2821SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2822 SDLoc DL(N);
2823 EVT VT = N->getValueType(0);
2824 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2825 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2826 // Round the value to the desired precision (that of the source type).
2827 return DAG.getNode(
2828 ISD::FP_EXTEND, DL, NVT,
2829 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2830 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2831}
2832
2833SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2834 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2835 N->getValueType(0)));
2836}
2837
2838SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2839 // Expand and promote recursively.
2840 // TODO: This is non-optimal, but dealing with the concurrently happening
2841 // vector-legalization is non-trivial. We could do something similar to
2842 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2843 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2844 return SDValue();
2845}
2846
2847SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2848 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2849 return SDValue();
2850}
2851
2852SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2853 EVT VT = N->getValueType(0);
2854
2855 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2856 SDLoc SL(N);
2857
2858 SDValue CastVal = BitConvertToInteger(AM->getVal());
2859 EVT CastVT = CastVal.getValueType();
2860
2861 SDValue NewAtomic
2862 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2863 DAG.getVTList(CastVT, MVT::Other),
2864 { AM->getChain(), AM->getBasePtr(), CastVal },
2865 AM->getMemOperand());
2866
2867 SDValue Result = NewAtomic;
2868
2869 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2870 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2871 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2872 NewAtomic);
2873 }
2874
2875 // Legalize the chain result by replacing uses of the old value chain with the
2876 // new one
2877 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2878
2879 return Result;
2880
2881}
2882
2883//===----------------------------------------------------------------------===//
2884// Half Result Soft Promotion
2885//===----------------------------------------------------------------------===//
2886
2887void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2888 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2889 N->dump(&DAG));
2890 SDValue R = SDValue();
2891
2892 // See if the target wants to custom expand this node.
2893 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2894 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2895 return;
2896 }
2897
2898 switch (N->getOpcode()) {
2899 default:
2900#ifndef NDEBUG
2901 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2902 N->dump(&DAG); dbgs() << "\n";
2903#endif
2904 report_fatal_error("Do not know how to soft promote this operator's "
2905 "result!");
2906
2907 case ISD::ARITH_FENCE:
2908 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2909 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2910 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2911 case ISD::EXTRACT_VECTOR_ELT:
2912 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2913 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2914 case ISD::STRICT_FP_ROUND:
2915 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2916
2917 // Unary FP Operations
2918 case ISD::FABS:
2919 case ISD::FCBRT:
2920 case ISD::FCEIL:
2921 case ISD::FCOS:
2922 case ISD::FEXP:
2923 case ISD::FEXP2:
2924 case ISD::FEXP10:
2925 case ISD::FFLOOR:
2926 case ISD::FLOG:
2927 case ISD::FLOG2:
2928 case ISD::FLOG10:
2929 case ISD::FNEARBYINT:
2930 case ISD::FNEG:
2931 case ISD::FREEZE:
2932 case ISD::FRINT:
2933 case ISD::FROUND:
2934 case ISD::FROUNDEVEN:
2935 case ISD::FSIN:
2936 case ISD::FSQRT:
2937 case ISD::FTRUNC:
2938 case ISD::FTAN:
2939 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2940
2941 // Binary FP Operations
2942 case ISD::FADD:
2943 case ISD::FDIV:
2944 case ISD::FMAXIMUM:
2945 case ISD::FMINIMUM:
2946 case ISD::FMAXNUM:
2947 case ISD::FMINNUM:
2948 case ISD::FMUL:
2949 case ISD::FPOW:
2950 case ISD::FREM:
2951 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2952
2953 case ISD::FMA: // FMA is same as FMAD
2954 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2955
2956 case ISD::FPOWI:
2957 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2958
2959 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2960
2961 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2962 case ISD::ATOMIC_LOAD:
2963 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2964 break;
2965 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2966 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2967 case ISD::SINT_TO_FP:
2968 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2969 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2970 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2971 case ISD::VECREDUCE_FADD:
2972 case ISD::VECREDUCE_FMUL:
2973 case ISD::VECREDUCE_FMIN:
2974 case ISD::VECREDUCE_FMAX:
2975 case ISD::VECREDUCE_FMAXIMUM:
2976 case ISD::VECREDUCE_FMINIMUM:
2977 R = SoftPromoteHalfRes_VECREDUCE(N);
2978 break;
2979 case ISD::VECREDUCE_SEQ_FADD:
2980 case ISD::VECREDUCE_SEQ_FMUL:
2981 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2982 break;
2983 }
2984
2985 if (R.getNode())
2986 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2987}
2988
2989SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2990 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
2991 BitConvertToInteger(N->getOperand(0)));
2992}
2993
2994SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2995 return BitConvertToInteger(N->getOperand(0));
2996}
2997
2998SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2999 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3000
3001 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3002 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3003 MVT::i16);
3004}
3005
3006SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3007 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3008 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3009 NewOp.getValueType().getVectorElementType(), NewOp,
3010 N->getOperand(1));
3011}
3012
3013SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3014 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3015 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3016 SDLoc dl(N);
3017
3018 EVT LVT = LHS.getValueType();
3019 EVT RVT = RHS.getValueType();
3020
3021 unsigned LSize = LVT.getSizeInBits();
3022 unsigned RSize = RVT.getSizeInBits();
3023
3024 // First get the sign bit of second operand.
3025 SDValue SignBit = DAG.getNode(
3026 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3027 DAG.getConstant(RSize - 1, dl,
3028 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3029 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3030
3031 // Shift right or sign-extend it if the two operands have different types.
3032 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3033 if (SizeDiff > 0) {
3034 SignBit =
3035 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3036 DAG.getConstant(SizeDiff, dl,
3037 TLI.getShiftAmountTy(SignBit.getValueType(),
3038 DAG.getDataLayout())));
3039 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3040 } else if (SizeDiff < 0) {
3041 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3042 SignBit =
3043 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3044 DAG.getConstant(-SizeDiff, dl,
3045 TLI.getShiftAmountTy(SignBit.getValueType(),
3046 DAG.getDataLayout())));
3047 }
3048
3049 // Clear the sign bit of the first operand.
3050 SDValue Mask = DAG.getNode(
3051 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3052 DAG.getConstant(LSize - 1, dl,
3053 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3054 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3055 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3056
3057 // Or the value with the sign bit.
3058 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3059}
3060
3061SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3062 EVT OVT = N->getValueType(0);
3063 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3064 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3065 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3066 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3067 SDLoc dl(N);
3068
3069 // Promote to the larger FP type.
3070 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3071 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3072 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3073 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3074
3075 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3076
3077 // Convert back to FP16 as an integer.
3078 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3079}
3080
3081SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3082 EVT OVT = N->getValueType(0);
3083 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3084 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3085 SDValue Op1 = N->getOperand(1);
3086 SDLoc dl(N);
3087
3088 // Promote to the larger FP type.
3089 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3090
3091 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3092
3093 // Convert back to FP16 as an integer.
3094 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3095}
3096
3097SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3098 EVT OVT = N->getValueType(0);
3099 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3100 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3101 SDLoc dl(N);
3102
3103 // Promote to the larger FP type.
3104 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3105
3106 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3107 DAG.getVTList(NVT, N->getValueType(1)), Op);
3108
3109 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3110
3111 // Convert back to FP16 as an integer.
3112 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3116 EVT RVT = N->getValueType(0);
3117 EVT SVT = N->getOperand(0).getValueType();
3118
3119 if (N->isStrictFPOpcode()) {
3120 // FIXME: assume we only have two f16 variants for now.
3121 unsigned Opcode;
3122 if (RVT == MVT::f16)
3123 Opcode = ISD::STRICT_FP_TO_FP16;
3124 else if (RVT == MVT::bf16)
3125 Opcode = ISD::STRICT_FP_TO_BF16;
3126 else
3127 llvm_unreachable("unknown half type");
3128 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3129 {N->getOperand(0), N->getOperand(1)});
3130 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3131 return Res;
3132 }
3133
3134 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3135 N->getOperand(0));
3136}
3137
3138SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3139 LoadSDNode *L = cast<LoadSDNode>(N);
3140
3141 // Load the value as an integer value with the same number of bits.
3142 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3143 SDValue NewL =
3144 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3145 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3146 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3147 L->getMemOperand()->getFlags(), L->getAAInfo());
3148 // Legalize the chain result by replacing uses of the old value chain with the
3149 // new one
3150 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3151 return NewL;
3152}
3153
3154SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3155 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3156
3157 // Load the value as an integer value with the same number of bits.
3158 SDValue NewL = DAG.getAtomic(
3159 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3160 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3161
3162 // Legalize the chain result by replacing uses of the old value chain with the
3163 // new one
3164 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3165 return NewL;
3166}
3167
3168SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3169 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3170 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3171 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3172 Op2);
3173}
3174
3175SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3176 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3177 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3178 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3179 N->getOperand(0), N->getOperand(1), Op2, Op3,
3180 N->getOperand(4));
3181}
3182
3183SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3184 EVT OVT = N->getValueType(0);
3185 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3186 SDLoc dl(N);
3187
3188 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3189
3190 // Round the value to the softened type.
3191 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3192}
3193
3194SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3195 return DAG.getUNDEF(MVT::i16);
3196}
3197
3198SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3199 EVT OVT = N->getValueType(0);
3200 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3201 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3202 SDLoc dl(N);
3203
3204 // Promote to the larger FP type.
3205 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3206
3207 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3208
3209 // Convert back to FP16 as an integer.
3210 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3211}
3212
3213SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3214 EVT OVT = N->getValueType(0);
3215 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3216 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3217 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3218 SDLoc dl(N);
3219
3220 // Promote to the larger FP type.
3221 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3222 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3223 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3224
3225 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3226
3227 // Convert back to FP16 as an integer.
3228 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3229}
3230
3231SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3232 // Expand and soften recursively.
3233 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3234 return SDValue();
3235}
3236
3237SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3238 // Expand and soften.
3239 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3240 return SDValue();
3241}
3242
3243//===----------------------------------------------------------------------===//
3244// Half Operand Soft Promotion
3245//===----------------------------------------------------------------------===//
3246
3247bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3248 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3249 N->dump(&DAG));
3250 SDValue Res = SDValue();
3251
3252 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3253 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3254 return false;
3255 }
3256
3257 // Nodes that use a promotion-requiring floating point operand, but doesn't
3258 // produce a soft promotion-requiring floating point result, need to be
3259 // legalized to use the soft promoted float operand. Nodes that produce at
3260 // least one soft promotion-requiring floating point result have their
3261 // operands legalized as a part of PromoteFloatResult.
3262 switch (N->getOpcode()) {
3263 default:
3264 #ifndef NDEBUG
3265 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3266 N->dump(&DAG); dbgs() << "\n";
3267 #endif
3268 report_fatal_error("Do not know how to soft promote this operator's "
3269 "operand!");
3270
3271 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3272 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3273 case ISD::FP_TO_SINT:
3274 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3275 case ISD::FP_TO_SINT_SAT:
3276 case ISD::FP_TO_UINT_SAT:
3277 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3278 case ISD::STRICT_FP_EXTEND:
3279 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3280 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3281 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3282 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3283 case ISD::ATOMIC_STORE:
3284 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3285 break;
3286 case ISD::STACKMAP:
3287 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3288 break;
3289 case ISD::PATCHPOINT:
3290 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3291 break;
3292 }
3293
3294 if (!Res.getNode())
3295 return false;
3296
3297 assert(Res.getNode() != N && "Expected a new node!");
3298
3299 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3300 "Invalid operand expansion");
3301
3302 ReplaceValueWith(SDValue(N, 0), Res);
3303 return false;
3304}
3305
3306SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3307 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3308
3309 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3310}
3311
3312SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3313 unsigned OpNo) {
3314 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3315 SDValue Op1 = N->getOperand(1);
3316 EVT RVT = Op1.getValueType();
3317 SDLoc dl(N);
3318
3319 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3320
3321 Op1 = GetSoftPromotedHalf(Op1);
3322 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3323
3324 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3325 Op1);
3326}
3327
3328SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3329 EVT RVT = N->getValueType(0);
3330 bool IsStrict = N->isStrictFPOpcode();
3331 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3332 EVT SVT = Op.getValueType();
3333 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3334
3335 if (IsStrict) {
3336 unsigned Opcode;
3337 if (SVT == MVT::f16)
3338 Opcode = ISD::STRICT_FP16_TO_FP;
3339 else if (SVT == MVT::bf16)
3340 Opcode = ISD::STRICT_BF16_TO_FP;
3341 else
3342 llvm_unreachable("unknown half type");
3343 SDValue Res =
3344 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3345 {N->getOperand(0), Op});
3346 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3347 ReplaceValueWith(SDValue(N, 0), Res);
3348 return SDValue();
3349 }
3350
3351 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3352}
3353
3354SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3355 EVT RVT = N->getValueType(0);
3356 SDValue Op = N->getOperand(0);
3357 EVT SVT = Op.getValueType();
3358 SDLoc dl(N);
3359
3360 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3361
3362 Op = GetSoftPromotedHalf(Op);
3363
3364 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3365
3366 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3367}
3368
3369SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3370 EVT RVT = N->getValueType(0);
3371 SDValue Op = N->getOperand(0);
3372 EVT SVT = Op.getValueType();
3373 SDLoc dl(N);
3374
3375 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3376
3377 Op = GetSoftPromotedHalf(Op);
3378
3379 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3380
3381 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3382 N->getOperand(1));
3383}
3384
3385SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3386 unsigned OpNo) {
3387 assert(OpNo == 0 && "Can only soften the comparison values");
3388 SDValue Op0 = N->getOperand(0);
3389 SDValue Op1 = N->getOperand(1);
3390 SDLoc dl(N);
3391
3392 EVT SVT = Op0.getValueType();
3393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3394
3395 Op0 = GetSoftPromotedHalf(Op0);
3396 Op1 = GetSoftPromotedHalf(Op1);
3397
3398 // Promote to the larger FP type.
3399 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3400 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3401 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3402
3403 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3404 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3405}
3406
3407SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3408 SDValue Op0 = N->getOperand(0);
3409 SDValue Op1 = N->getOperand(1);
3410 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3411 SDLoc dl(N);
3412
3413 EVT SVT = Op0.getValueType();
3414 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3415
3416 Op0 = GetSoftPromotedHalf(Op0);
3417 Op1 = GetSoftPromotedHalf(Op1);
3418
3419 // Promote to the larger FP type.
3420 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3421 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3422 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3423
3424 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3425}
3426
3427SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3428 assert(OpNo == 1 && "Can only soften the stored value!");
3429 StoreSDNode *ST = cast<StoreSDNode>(N);
3430 SDValue Val = ST->getValue();
3431 SDLoc dl(N);
3432
3433 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3434 SDValue Promoted = GetSoftPromotedHalf(Val);
3435 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3436 ST->getMemOperand());
3437}
3438
3439SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3440 unsigned OpNo) {
3441 assert(OpNo == 1 && "Can only soften the stored value!");
3442 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3443 SDValue Val = ST->getVal();
3444 SDLoc dl(N);
3445
3446 SDValue Promoted = GetSoftPromotedHalf(Val);
3447 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3448 ST->getChain(), Promoted, ST->getBasePtr(),
3449 ST->getMemOperand());
3450}
3451
3452SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3453 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3454 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3455 SDValue Op = N->getOperand(OpNo);
3456 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3457 SDValue NewNode =
3458 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3459
3460 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3461 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3462
3463 return SDValue(); // Signal that we replaced the node ourselves.
3464}
3465
3466SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3467 unsigned OpNo) {
3468 assert(OpNo >= 7);
3469 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3470 SDValue Op = N->getOperand(OpNo);
3471 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3472 SDValue NewNode =
3473 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3474
3475 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3476 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3477
3478 return SDValue(); // Signal that we replaced the node ourselves.
3479}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:74
Expansion
DXIL Intrinsic Expansion
Definition: DXILIntrinsicExpansion.cpp:323
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:354
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:1071
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2241
GetPromotionOpcodeStrict
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2255
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1254
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:77
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:213
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1386
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:208
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:548
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1496
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1518
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1718
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:239
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:286
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2430
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:53
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:144
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:146
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1406
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1151
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:477
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:1028
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:1031
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1186
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8866
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10290
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1227
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1809
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8849
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8570
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1118
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2369
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1256
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:486
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1622
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9907
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8950
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8899
llvm::SelectionDAG::EVTToAPFloatSemantics
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Definition: SelectionDAG.h:1821
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1266
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1748
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9913
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:493
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:481
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:2015
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:499
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
Definition: SelectionDAG.cpp:1753
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2488
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10380
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:568
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2458
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:564
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:1055
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1143
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3431
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:291
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:146
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:11002
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:11053
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:10962
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:8332
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:764
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:484
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1360
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1262
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:434
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:495
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1052
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:797
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:491
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:804
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1376
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
llvm::ISD::VECREDUCE_FMAXIMUM
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1380
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:485
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:927
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:429
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:917
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:954
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:418
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:788
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:458
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1373
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1377
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:439
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:944
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1098
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:508
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:741
llvm::ISD::STRICT_FPOWI
@ STRICT_FPOWI
Definition: ISDOpcodes.h:420
llvm::ISD::ATOMIC_LOAD
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1258
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:218
llvm::ISD::EXTRACT_ELEMENT
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:229
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:438
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1246
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:930
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:929
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:719
llvm::ISD::FMINNUM_IEEE
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:999
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:432
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:543
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:794
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:870
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:433
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:756
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:986
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:441
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:902
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:436
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:457
llvm::ISD::STRICT_BF16_TO_FP
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:938
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:435
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:437
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:936
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:451
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:473
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:450
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1005
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:850
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:478
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:694
llvm::ISD::STRICT_FP_TO_BF16
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:939
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1374
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:407
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:428
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:442
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:426
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1283
llvm::ISD::FFREXP
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:959
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:883
llvm::ISD::STRICT_FLDEXP
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:421
llvm::ISD::STRICT_LLROUND
@ STRICT_LLROUND
Definition: ISDOpcodes.h:440
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:431
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:869
llvm::ISD::VECREDUCE_FMINIMUM
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1381
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:800
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1167
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1361
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:501
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:430
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:523
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3214
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3207
llvm::ISD::isUNINDEXEDStore
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
Definition: SelectionDAGNodes.h:3221
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1554
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3176
llvm::RTLIB::getPOWI
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:580
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:490
llvm::RTLIB::getLDEXP
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:585
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:536
llvm::RTLIB::getFREXP
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:590
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:441
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:392
llvm::RTLIB::getFPEXT
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:314
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:349
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:137
llvm::tgtok::FalseVal
@ FalseVal
Definition: TGLexer.h:59
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Offset
@ Offset
Definition: DWP.cpp:480
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
llvm::Op
DWARFExpression::Operation Op
Definition: DWARFExpression.cpp:22
raw_ostream.h
N
#define N
llvm::APFloatBase::PPCDoubleDouble
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:274
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::EVT::bitsGE
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1062
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::TargetLowering::MakeLibCallOptions
This structure is used to pass arguments to makeLibCall function.
Definition: TargetLowering.h:4697
llvm::TargetLowering::MakeLibCallOptions::setSExt
MakeLibCallOptions & setSExt(bool Value=true)
Definition: TargetLowering.h:4712
llvm::TargetLowering::MakeLibCallOptions::setTypeListBeforeSoften
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
Definition: TargetLowering.h:4732
Generated on Sat Jun 29 2024 07:36:17 for LLVM by doxygen 1.9.6