LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::STRICT_FCEIL:
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::STRICT_FCOS:
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::STRICT_FDIV:
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::STRICT_FEXP:
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::STRICT_FEXP2:
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::STRICT_FFLOOR:
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::STRICT_FLOG:
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::STRICT_FLOG2:
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::STRICT_FLOG10:
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::STRICT_FNEARBYINT:
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::STRICT_FP_EXTEND:
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::STRICT_FP_ROUND:
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::STRICT_FPOWI:
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
122 case ISD::STRICT_FRINT:
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
124 case ISD::STRICT_FROUND:
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
126 case ISD::STRICT_FROUNDEVEN:
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
130 case ISD::STRICT_FSQRT:
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
134 case ISD::STRICT_FTRUNC:
135 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
136 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
137 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
138 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
139 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
140 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
141 case ISD::STRICT_SINT_TO_FP:
142 case ISD::STRICT_UINT_TO_FP:
143 case ISD::SINT_TO_FP:
144 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
145 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
146 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
147 case ISD::VECREDUCE_FADD:
148 case ISD::VECREDUCE_FMUL:
149 case ISD::VECREDUCE_FMIN:
150 case ISD::VECREDUCE_FMAX:
151 case ISD::VECREDUCE_FMAXIMUM:
152 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
153 case ISD::VECREDUCE_SEQ_FADD:
154 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
155 // clang-format on
156 }
157
158 // If R is null, the sub-method took care of registering the result.
159 if (R.getNode()) {
160 assert(R.getNode() != N);
161 SetSoftenedFloat(SDValue(N, ResNo), R);
162 }
163}
164
165SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
166 bool IsStrict = N->isStrictFPOpcode();
167 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
168 unsigned Offset = IsStrict ? 1 : 0;
169 assert(N->getNumOperands() == (1 + Offset) &&
170 "Unexpected number of operands!");
171 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
172 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
173 TargetLowering::MakeLibCallOptions CallOptions;
174 EVT OpVT = N->getOperand(0 + Offset).getValueType();
175 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
176 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
177 CallOptions, SDLoc(N),
178 Chain);
179 if (IsStrict)
180 ReplaceValueWith(SDValue(N, 1), Tmp.second);
181 return Tmp.first;
182}
183
184SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
185 bool IsStrict = N->isStrictFPOpcode();
186 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
187 unsigned Offset = IsStrict ? 1 : 0;
188 assert(N->getNumOperands() == (2 + Offset) &&
189 "Unexpected number of operands!");
190 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
191 GetSoftenedFloat(N->getOperand(1 + Offset)) };
192 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
193 TargetLowering::MakeLibCallOptions CallOptions;
194 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
195 N->getOperand(1 + Offset).getValueType() };
196 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
197 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
198 CallOptions, SDLoc(N),
199 Chain);
200 if (IsStrict)
201 ReplaceValueWith(SDValue(N, 1), Tmp.second);
202 return Tmp.first;
203}
204
205SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
206 return BitConvertToInteger(N->getOperand(0));
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
210 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
211 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
212 GetSoftenedFloat(N->getOperand(0)));
213}
214
215SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
216 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
217 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
218 GetSoftenedFloat(N->getOperand(0)));
219 return NewFence;
220}
221
222SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
223 unsigned ResNo) {
224 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
225 return BitConvertToInteger(Op);
226}
227
228SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
229 // Convert the inputs to integers, and build a new pair out of them.
230 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
231 TLI.getTypeToTransformTo(*DAG.getContext(),
232 N->getValueType(0)),
233 BitConvertToInteger(N->getOperand(0)),
234 BitConvertToInteger(N->getOperand(1)));
235}
236
237SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
238 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
239 // In ppcf128, the high 64 bits are always first in memory regardless
240 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
241 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
242 // way. However, APInt's are serialized in an Endian-sensitive fashion,
243 // so on big-Endian targets, the two doubles are output in the wrong
244 // order. Fix this by manually flipping the order of the high 64 bits
245 // and the low 64 bits here.
246 if (DAG.getDataLayout().isBigEndian() &&
247 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
248 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
249 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
250 APInt Val(128, words);
251 return DAG.getConstant(Val, SDLoc(CN),
252 TLI.getTypeToTransformTo(*DAG.getContext(),
253 CN->getValueType(0)));
254 } else {
255 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
256 TLI.getTypeToTransformTo(*DAG.getContext(),
257 CN->getValueType(0)));
258 }
259}
260
261SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
262 SDValue Src = N->getOperand(0);
263 assert(Src.getValueType() == MVT::ppcf128 &&
264 "In floats only ppcf128 can be extracted by element!");
265 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
266 N->getValueType(0).changeTypeToInteger(),
267 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
268}
269
270SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
271 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
272 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
273 NewOp.getValueType().getVectorElementType(),
274 NewOp, N->getOperand(1));
275}
276
277SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
278 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
279 unsigned Size = NVT.getSizeInBits();
280
281 // Mask = ~(1 << (Size-1))
282 APInt API = APInt::getAllOnes(Size);
283 API.clearBit(Size - 1);
284 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
285 SDValue Op = GetSoftenedFloat(N->getOperand(0));
286 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
287}
288
289SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
290 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
291 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
292 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
293 RTLIB::FMIN_F32,
294 RTLIB::FMIN_F64,
295 RTLIB::FMIN_F80,
296 RTLIB::FMIN_F128,
297 RTLIB::FMIN_PPCF128));
298}
299
300SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
301 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
302 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
303 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
304 RTLIB::FMAX_F32,
305 RTLIB::FMAX_F64,
306 RTLIB::FMAX_F80,
307 RTLIB::FMAX_F128,
308 RTLIB::FMAX_PPCF128));
309}
310
311SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
312 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
313 RTLIB::ADD_F32,
314 RTLIB::ADD_F64,
315 RTLIB::ADD_F80,
316 RTLIB::ADD_F128,
317 RTLIB::ADD_PPCF128));
318}
319
320SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
321 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
322 RTLIB::CBRT_F32,
323 RTLIB::CBRT_F64,
324 RTLIB::CBRT_F80,
325 RTLIB::CBRT_F128,
326 RTLIB::CBRT_PPCF128));
327}
328
329SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
330 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
331 RTLIB::CEIL_F32,
332 RTLIB::CEIL_F64,
333 RTLIB::CEIL_F80,
334 RTLIB::CEIL_F128,
335 RTLIB::CEIL_PPCF128));
336}
337
338SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
339 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
340 SDValue RHS = BitConvertToInteger(N->getOperand(1));
341 SDLoc dl(N);
342
343 EVT LVT = LHS.getValueType();
344 EVT RVT = RHS.getValueType();
345
346 unsigned LSize = LVT.getSizeInBits();
347 unsigned RSize = RVT.getSizeInBits();
348
349 // First get the sign bit of second operand.
350 SDValue SignBit = DAG.getNode(
351 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
352 DAG.getConstant(RSize - 1, dl,
353 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
354 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
355
356 // Shift right or sign-extend it if the two operands have different types.
357 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
358 if (SizeDiff > 0) {
359 SignBit =
360 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
361 DAG.getConstant(SizeDiff, dl,
362 TLI.getShiftAmountTy(SignBit.getValueType(),
363 DAG.getDataLayout())));
364 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
365 } else if (SizeDiff < 0) {
366 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
367 SignBit =
368 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
369 DAG.getConstant(-SizeDiff, dl,
370 TLI.getShiftAmountTy(SignBit.getValueType(),
371 DAG.getDataLayout())));
372 }
373
374 // Clear the sign bit of the first operand.
375 SDValue Mask = DAG.getNode(
376 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
377 DAG.getConstant(LSize - 1, dl,
378 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
379 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
380 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
381
382 // Or the value with the sign bit.
383 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
384}
385
386SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
387 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
388 RTLIB::COS_F32,
389 RTLIB::COS_F64,
390 RTLIB::COS_F80,
391 RTLIB::COS_F128,
392 RTLIB::COS_PPCF128));
393}
394
395SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
396 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
397 RTLIB::DIV_F32,
398 RTLIB::DIV_F64,
399 RTLIB::DIV_F80,
400 RTLIB::DIV_F128,
401 RTLIB::DIV_PPCF128));
402}
403
404SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
405 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
406 RTLIB::EXP_F32,
407 RTLIB::EXP_F64,
408 RTLIB::EXP_F80,
409 RTLIB::EXP_F128,
410 RTLIB::EXP_PPCF128));
411}
412
413SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
414 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
415 RTLIB::EXP2_F32,
416 RTLIB::EXP2_F64,
417 RTLIB::EXP2_F80,
418 RTLIB::EXP2_F128,
419 RTLIB::EXP2_PPCF128));
420}
421
422SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
423 return SoftenFloatRes_Unary(
424 N,
425 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
426 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
427}
428
429SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
430 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
431 RTLIB::FLOOR_F32,
432 RTLIB::FLOOR_F64,
433 RTLIB::FLOOR_F80,
434 RTLIB::FLOOR_F128,
435 RTLIB::FLOOR_PPCF128));
436}
437
438SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
439 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
440 RTLIB::LOG_F32,
441 RTLIB::LOG_F64,
442 RTLIB::LOG_F80,
443 RTLIB::LOG_F128,
444 RTLIB::LOG_PPCF128));
445}
446
447SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
448 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
449 RTLIB::LOG2_F32,
450 RTLIB::LOG2_F64,
451 RTLIB::LOG2_F80,
452 RTLIB::LOG2_F128,
453 RTLIB::LOG2_PPCF128));
454}
455
456SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
457 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
458 RTLIB::LOG10_F32,
459 RTLIB::LOG10_F64,
460 RTLIB::LOG10_F80,
461 RTLIB::LOG10_F128,
462 RTLIB::LOG10_PPCF128));
463}
464
465SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
466 bool IsStrict = N->isStrictFPOpcode();
467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468 unsigned Offset = IsStrict ? 1 : 0;
469 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
470 GetSoftenedFloat(N->getOperand(1 + Offset)),
471 GetSoftenedFloat(N->getOperand(2 + Offset)) };
472 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
473 TargetLowering::MakeLibCallOptions CallOptions;
474 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
475 N->getOperand(1 + Offset).getValueType(),
476 N->getOperand(2 + Offset).getValueType() };
477 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
478 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
479 GetFPLibCall(N->getValueType(0),
480 RTLIB::FMA_F32,
481 RTLIB::FMA_F64,
482 RTLIB::FMA_F80,
483 RTLIB::FMA_F128,
484 RTLIB::FMA_PPCF128),
485 NVT, Ops, CallOptions, SDLoc(N), Chain);
486 if (IsStrict)
487 ReplaceValueWith(SDValue(N, 1), Tmp.second);
488 return Tmp.first;
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
492 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
493 RTLIB::MUL_F32,
494 RTLIB::MUL_F64,
495 RTLIB::MUL_F80,
496 RTLIB::MUL_F128,
497 RTLIB::MUL_PPCF128));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
501 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
502 RTLIB::NEARBYINT_F32,
503 RTLIB::NEARBYINT_F64,
504 RTLIB::NEARBYINT_F80,
505 RTLIB::NEARBYINT_F128,
506 RTLIB::NEARBYINT_PPCF128));
507}
508
509SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
511 SDLoc dl(N);
512
513 // Expand Y = FNEG(X) -> Y = X ^ sign mask
514 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
515 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
516 DAG.getConstant(SignMask, dl, NVT));
517}
518
519SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
520 bool IsStrict = N->isStrictFPOpcode();
521 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
522 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
523
524 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
525
526 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
527 Op = GetPromotedFloat(Op);
528 // If the promotion did the FP_EXTEND to the destination type for us,
529 // there's nothing left to do here.
530 if (Op.getValueType() == N->getValueType(0)) {
531 if (IsStrict)
532 ReplaceValueWith(SDValue(N, 1), Chain);
533 return BitConvertToInteger(Op);
534 }
535 }
536
537 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
538 // -> f32, so proceed in two stages. Also, it's entirely possible for both
539 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
540 // than FP16_TO_FP.
541 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
542 N->getValueType(0) != MVT::f32) {
543 if (IsStrict) {
544 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
545 { MVT::f32, MVT::Other }, { Chain, Op });
546 Chain = Op.getValue(1);
547 } else {
548 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
549 }
550 }
551
552 if (Op.getValueType() == MVT::bf16) {
553 // FIXME: Need ReplaceValueWith on chain in strict case
554 return SoftenFloatRes_BF16_TO_FP(N);
555 }
556
557 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
558 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
559 TargetLowering::MakeLibCallOptions CallOptions;
560 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
561 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
563 CallOptions, SDLoc(N),
564 Chain);
565 if (IsStrict)
566 ReplaceValueWith(SDValue(N, 1), Tmp.second);
567 return Tmp.first;
568}
569
570// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
571// nodes?
572SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
573 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
574 SDValue Op = N->getOperand(0);
575 TargetLowering::MakeLibCallOptions CallOptions;
576 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
577 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
578 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
579 CallOptions, SDLoc(N)).first;
580 if (N->getValueType(0) == MVT::f32)
581 return Res32;
582
583 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
585 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
586 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
587}
588
589// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
590// nodes?
591SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
592 assert(N->getValueType(0) == MVT::f32 &&
593 "Can only soften BF16_TO_FP with f32 result");
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
595 SDValue Op = N->getOperand(0);
596 SDLoc DL(N);
597 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
598 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
599 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
600 DAG.getShiftAmountConstant(16, NVT, DL));
601 return Res;
602}
603
604SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
605 bool IsStrict = N->isStrictFPOpcode();
606 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
607 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
608 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
609 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
610 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
611 TargetLowering::MakeLibCallOptions CallOptions;
612 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
613 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
614 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
615 CallOptions, SDLoc(N),
616 Chain);
617 if (IsStrict)
618 ReplaceValueWith(SDValue(N, 1), Tmp.second);
619 return Tmp.first;
620}
621
622SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
623 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
624 RTLIB::POW_F32,
625 RTLIB::POW_F64,
626 RTLIB::POW_F80,
627 RTLIB::POW_F128,
628 RTLIB::POW_PPCF128));
629}
630
631SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
632 bool IsStrict = N->isStrictFPOpcode();
633 unsigned Offset = IsStrict ? 1 : 0;
634 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
635 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
636 "Unsupported power type!");
637 bool IsPowI =
638 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
639
640 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
641 : RTLIB::getLDEXP(N->getValueType(0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
643 if (!TLI.getLibcallName(LC)) {
644 // Some targets don't have a powi libcall; use pow instead.
645 // FIXME: Implement this if some target needs it.
646 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
647 return DAG.getUNDEF(N->getValueType(0));
648 }
649
650 if (DAG.getLibInfo().getIntSize() !=
651 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
652 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
653 // would use the wrong type for the argument.
654 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
655 return DAG.getUNDEF(N->getValueType(0));
656 }
657
658 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
659 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
660 N->getOperand(1 + Offset) };
661 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
662 TargetLowering::MakeLibCallOptions CallOptions;
663 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
664 N->getOperand(1 + Offset).getValueType() };
665 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
666 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
667 CallOptions, SDLoc(N),
668 Chain);
669 if (IsStrict)
670 ReplaceValueWith(SDValue(N, 1), Tmp.second);
671 return Tmp.first;
672}
673
674SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
675 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
676 EVT VT0 = N->getValueType(0);
677 EVT VT1 = N->getValueType(1);
678 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
679
680 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
681 // If the exponent does not match with sizeof(int) a libcall would use the
682 // wrong type for the argument.
683 // TODO: Should be able to handle mismatches.
684 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
685 return DAG.getUNDEF(N->getValueType(0));
686 }
687
688 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
689 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
690
691 SDLoc DL(N);
692
693 TargetLowering::MakeLibCallOptions CallOptions;
694 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
695 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
696
697 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
698 // but we only really need to handle the 0th one for softening anyway.
699 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
700
701 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
702 /*Chain=*/SDValue());
703 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
704 auto PtrInfo =
705 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
706
707 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
708
709 ReplaceValueWith(SDValue(N, 1), LoadExp);
710 return ReturnVal;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
714 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
715 RTLIB::REM_F32,
716 RTLIB::REM_F64,
717 RTLIB::REM_F80,
718 RTLIB::REM_F128,
719 RTLIB::REM_PPCF128));
720}
721
722SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
723 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
724 RTLIB::RINT_F32,
725 RTLIB::RINT_F64,
726 RTLIB::RINT_F80,
727 RTLIB::RINT_F128,
728 RTLIB::RINT_PPCF128));
729}
730
731SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
732 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
733 RTLIB::ROUND_F32,
734 RTLIB::ROUND_F64,
735 RTLIB::ROUND_F80,
736 RTLIB::ROUND_F128,
737 RTLIB::ROUND_PPCF128));
738}
739
740SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
741 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
742 RTLIB::ROUNDEVEN_F32,
743 RTLIB::ROUNDEVEN_F64,
744 RTLIB::ROUNDEVEN_F80,
745 RTLIB::ROUNDEVEN_F128,
746 RTLIB::ROUNDEVEN_PPCF128));
747}
748
749SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
750 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
751 RTLIB::SIN_F32,
752 RTLIB::SIN_F64,
753 RTLIB::SIN_F80,
754 RTLIB::SIN_F128,
755 RTLIB::SIN_PPCF128));
756}
757
758SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
759 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
760 RTLIB::SQRT_F32,
761 RTLIB::SQRT_F64,
762 RTLIB::SQRT_F80,
763 RTLIB::SQRT_F128,
764 RTLIB::SQRT_PPCF128));
765}
766
767SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
768 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
769 RTLIB::SUB_F32,
770 RTLIB::SUB_F64,
771 RTLIB::SUB_F80,
772 RTLIB::SUB_F128,
773 RTLIB::SUB_PPCF128));
774}
775
776SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
777 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
778 RTLIB::TRUNC_F32,
779 RTLIB::TRUNC_F64,
780 RTLIB::TRUNC_F80,
781 RTLIB::TRUNC_F128,
782 RTLIB::TRUNC_PPCF128));
783}
784
785SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
786 LoadSDNode *L = cast<LoadSDNode>(N);
787 EVT VT = N->getValueType(0);
788 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
789 SDLoc dl(N);
790
791 auto MMOFlags =
792 L->getMemOperand()->getFlags() &
793 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
794 SDValue NewL;
795 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
796 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
797 L->getChain(), L->getBasePtr(), L->getOffset(),
798 L->getPointerInfo(), NVT, L->getOriginalAlign(),
799 MMOFlags, L->getAAInfo());
800 // Legalized the chain result - switch anything that used the old chain to
801 // use the new one.
802 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
803 return NewL;
804 }
805
806 // Do a non-extending load followed by FP_EXTEND.
807 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
808 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
809 L->getPointerInfo(), L->getMemoryVT(),
810 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
811 // Legalized the chain result - switch anything that used the old chain to
812 // use the new one.
813 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
814 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
815 return BitConvertToInteger(ExtendNode);
816}
817
818SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
819 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
820 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
821 return DAG.getSelect(SDLoc(N),
822 LHS.getValueType(), N->getOperand(0), LHS, RHS);
823}
824
825SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
826 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
827 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
828 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
829 LHS.getValueType(), N->getOperand(0),
830 N->getOperand(1), LHS, RHS, N->getOperand(4));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
834 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
835 N->getValueType(0)));
836}
837
838SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
839 SDValue Chain = N->getOperand(0); // Get the chain.
840 SDValue Ptr = N->getOperand(1); // Get the pointer.
841 EVT VT = N->getValueType(0);
842 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
843 SDLoc dl(N);
844
845 SDValue NewVAARG;
846 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
847 N->getConstantOperandVal(3));
848
849 // Legalized the chain result - switch anything that used the old chain to
850 // use the new one.
851 if (N != NewVAARG.getValue(1).getNode())
852 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
853 return NewVAARG;
854}
855
856SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
857 bool IsStrict = N->isStrictFPOpcode();
858 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
859 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
860 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
861 EVT RVT = N->getValueType(0);
862 EVT NVT = EVT();
863 SDLoc dl(N);
864
865 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
866 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
867 // match. Look for an appropriate libcall.
868 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
869 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
870 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
871 NVT = (MVT::SimpleValueType)t;
872 // The source needs to big enough to hold the operand.
873 if (NVT.bitsGE(SVT))
874 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
875 }
876 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
877
878 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
879 // Sign/zero extend the argument if the libcall takes a larger type.
880 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
881 NVT, N->getOperand(IsStrict ? 1 : 0));
882 TargetLowering::MakeLibCallOptions CallOptions;
883 CallOptions.setSExt(Signed);
884 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
885 std::pair<SDValue, SDValue> Tmp =
886 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
887 Op, CallOptions, dl, Chain);
888
889 if (IsStrict)
890 ReplaceValueWith(SDValue(N, 1), Tmp.second);
891 return Tmp.first;
892}
893
894SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
895 // Expand and soften recursively.
896 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
897 return SDValue();
898}
899
900SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
901 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
902 return SDValue();
903}
904
905//===----------------------------------------------------------------------===//
906// Convert Float Operand to Integer
907//===----------------------------------------------------------------------===//
908
909bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
910 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
911 SDValue Res = SDValue();
912
913 switch (N->getOpcode()) {
914 default:
915#ifndef NDEBUG
916 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
917 N->dump(&DAG); dbgs() << "\n";
918#endif
919 report_fatal_error("Do not know how to soften this operator's operand!");
920
921 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
922 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
923 case ISD::STRICT_FP_TO_FP16:
924 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
925 case ISD::FP_TO_BF16:
926 case ISD::STRICT_FP_TO_BF16:
927 case ISD::STRICT_FP_ROUND:
928 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
929 case ISD::STRICT_FP_TO_SINT:
930 case ISD::STRICT_FP_TO_UINT:
931 case ISD::FP_TO_SINT:
932 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
933 case ISD::FP_TO_SINT_SAT:
934 case ISD::FP_TO_UINT_SAT:
935 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
936 case ISD::STRICT_LROUND:
937 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
938 case ISD::STRICT_LLROUND:
939 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
940 case ISD::STRICT_LRINT:
941 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
942 case ISD::STRICT_LLRINT:
943 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
944 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
945 case ISD::STRICT_FSETCC:
946 case ISD::STRICT_FSETCCS:
947 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
948 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
949 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
950 }
951
952 // If the result is null, the sub-method took care of registering results etc.
953 if (!Res.getNode()) return false;
954
955 // If the result is N, the sub-method updated N in place. Tell the legalizer
956 // core about this to re-analyze.
957 if (Res.getNode() == N)
958 return true;
959
960 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
961 "Invalid operand softening");
962
963 ReplaceValueWith(SDValue(N, 0), Res);
964 return false;
965}
966
967SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
968 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
969
970 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
971}
972
973SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
974 // We actually deal with the partially-softened FP_TO_FP16 node too, which
975 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
976 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
977 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
978 N->getOpcode() == ISD::FP_TO_BF16 ||
979 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
980 N->getOpcode() == ISD::STRICT_FP_ROUND);
981
982 bool IsStrict = N->isStrictFPOpcode();
983 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
984 EVT SVT = Op.getValueType();
985 EVT RVT = N->getValueType(0);
986 EVT FloatRVT = RVT;
987 if (N->getOpcode() == ISD::FP_TO_FP16 ||
988 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
989 FloatRVT = MVT::f16;
990 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
991 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
992 FloatRVT = MVT::bf16;
993
994 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
995 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
996
997 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
998 Op = GetSoftenedFloat(Op);
999 TargetLowering::MakeLibCallOptions CallOptions;
1000 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1001 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1002 CallOptions, SDLoc(N),
1003 Chain);
1004 if (IsStrict) {
1005 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1006 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1007 return SDValue();
1008 }
1009 return Tmp.first;
1010}
1011
1012SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1013 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1014 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1015
1016 EVT VT = NewLHS.getValueType();
1017 NewLHS = GetSoftenedFloat(NewLHS);
1018 NewRHS = GetSoftenedFloat(NewRHS);
1019 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1020 N->getOperand(2), N->getOperand(3));
1021
1022 // If softenSetCCOperands returned a scalar, we need to compare the result
1023 // against zero to select between true and false values.
1024 if (!NewRHS.getNode()) {
1025 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1026 CCCode = ISD::SETNE;
1027 }
1028
1029 // Update N to have the operands specified.
1030 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1031 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1032 N->getOperand(4)),
1033 0);
1034}
1035
1036// Even if the result type is legal, no libcall may exactly match. (e.g. We
1037// don't have FP-i8 conversions) This helper method looks for an appropriate
1038// promoted libcall.
1039static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1040 bool Signed) {
1041 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1042 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1043 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1044 ++IntVT) {
1045 Promoted = (MVT::SimpleValueType)IntVT;
1046 // The type needs to big enough to hold the result.
1047 if (Promoted.bitsGE(RetVT))
1048 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1049 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1050 }
1051 return LC;
1052}
1053
1054SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1055 bool IsStrict = N->isStrictFPOpcode();
1056 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1058
1059 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1060 EVT SVT = Op.getValueType();
1061 EVT RVT = N->getValueType(0);
1062 EVT NVT = EVT();
1063 SDLoc dl(N);
1064
1065 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1066 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1067 // match, eg. we don't have fp -> i8 conversions.
1068 // Look for an appropriate libcall.
1069 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1070 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1071 "Unsupported FP_TO_XINT!");
1072
1073 Op = GetSoftenedFloat(Op);
1074 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1075 TargetLowering::MakeLibCallOptions CallOptions;
1076 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1077 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1078 CallOptions, dl, Chain);
1079
1080 // Truncate the result if the libcall returns a larger type.
1081 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1082
1083 if (!IsStrict)
1084 return Res;
1085
1086 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1087 ReplaceValueWith(SDValue(N, 0), Res);
1088 return SDValue();
1089}
1090
1091SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1092 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1093 return Res;
1094}
1095
1096SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1097 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1098 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1099
1100 EVT VT = NewLHS.getValueType();
1101 NewLHS = GetSoftenedFloat(NewLHS);
1102 NewRHS = GetSoftenedFloat(NewRHS);
1103 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1104 N->getOperand(0), N->getOperand(1));
1105
1106 // If softenSetCCOperands returned a scalar, we need to compare the result
1107 // against zero to select between true and false values.
1108 if (!NewRHS.getNode()) {
1109 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1110 CCCode = ISD::SETNE;
1111 }
1112
1113 // Update N to have the operands specified.
1114 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1115 N->getOperand(2), N->getOperand(3),
1116 DAG.getCondCode(CCCode)),
1117 0);
1118}
1119
1120SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1121 bool IsStrict = N->isStrictFPOpcode();
1122 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1123 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1124 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1125 ISD::CondCode CCCode =
1126 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1127
1128 EVT VT = Op0.getValueType();
1129 SDValue NewLHS = GetSoftenedFloat(Op0);
1130 SDValue NewRHS = GetSoftenedFloat(Op1);
1131 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1132 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1133
1134 // Update N to have the operands specified.
1135 if (NewRHS.getNode()) {
1136 if (IsStrict)
1137 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1138 NewRHS, DAG.getCondCode(CCCode));
1139 else
1140 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1141 DAG.getCondCode(CCCode)), 0);
1142 }
1143
1144 // Otherwise, softenSetCCOperands returned a scalar, use it.
1145 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1146 "Unexpected setcc expansion!");
1147
1148 if (IsStrict) {
1149 ReplaceValueWith(SDValue(N, 0), NewLHS);
1150 ReplaceValueWith(SDValue(N, 1), Chain);
1151 return SDValue();
1152 }
1153 return NewLHS;
1154}
1155
1156SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1157 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1158 assert(OpNo == 1 && "Can only soften the stored value!");
1159 StoreSDNode *ST = cast<StoreSDNode>(N);
1160 SDValue Val = ST->getValue();
1161 SDLoc dl(N);
1162
1163 if (ST->isTruncatingStore())
1164 // Do an FP_ROUND followed by a non-truncating store.
1165 Val = BitConvertToInteger(
1166 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1167 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1168 else
1169 Val = GetSoftenedFloat(Val);
1170
1171 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1172 ST->getMemOperand());
1173}
1174
1175SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1176 SDValue LHS = N->getOperand(0);
1177 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1178 SDLoc dl(N);
1179
1180 EVT LVT = LHS.getValueType();
1181 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1182 EVT RVT = RHS.getValueType();
1183
1184 unsigned LSize = LVT.getSizeInBits();
1185 unsigned RSize = RVT.getSizeInBits();
1186
1187 // Shift right or sign-extend it if the two operands have different types.
1188 int SizeDiff = RSize - LSize;
1189 if (SizeDiff > 0) {
1190 RHS =
1191 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1192 DAG.getConstant(SizeDiff, dl,
1193 TLI.getShiftAmountTy(RHS.getValueType(),
1194 DAG.getDataLayout())));
1195 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1196 } else if (SizeDiff < 0) {
1197 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1198 RHS =
1199 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1200 DAG.getConstant(-SizeDiff, dl,
1201 TLI.getShiftAmountTy(RHS.getValueType(),
1202 DAG.getDataLayout())));
1203 }
1204
1205 RHS = DAG.getBitcast(LVT, RHS);
1206 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1207}
1208
1209SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1210 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1211 bool IsStrict = N->isStrictFPOpcode();
1212 unsigned Offset = IsStrict ? 1 : 0;
1213 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1214 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1215 TargetLowering::MakeLibCallOptions CallOptions;
1216 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1217 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1218 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1219 CallOptions, SDLoc(N),
1220 Chain);
1221 if (IsStrict) {
1222 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1223 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1224 return SDValue();
1225 }
1226
1227 return Tmp.first;
1228}
1229
1230SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1231 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1232 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1233 RTLIB::LROUND_F32,
1234 RTLIB::LROUND_F64,
1235 RTLIB::LROUND_F80,
1236 RTLIB::LROUND_F128,
1237 RTLIB::LROUND_PPCF128));
1238}
1239
1240SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1241 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1242 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1243 RTLIB::LLROUND_F32,
1244 RTLIB::LLROUND_F64,
1245 RTLIB::LLROUND_F80,
1246 RTLIB::LLROUND_F128,
1247 RTLIB::LLROUND_PPCF128));
1248}
1249
1250SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1251 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1252 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1253 RTLIB::LRINT_F32,
1254 RTLIB::LRINT_F64,
1255 RTLIB::LRINT_F80,
1256 RTLIB::LRINT_F128,
1257 RTLIB::LRINT_PPCF128));
1258}
1259
1260SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1261 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1262 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1263 RTLIB::LLRINT_F32,
1264 RTLIB::LLRINT_F64,
1265 RTLIB::LLRINT_F80,
1266 RTLIB::LLRINT_F128,
1267 RTLIB::LLRINT_PPCF128));
1268}
1269
1270//===----------------------------------------------------------------------===//
1271// Float Result Expansion
1272//===----------------------------------------------------------------------===//
1273
1274/// ExpandFloatResult - This method is called when the specified result of the
1275/// specified node is found to need expansion. At this point, the node may also
1276/// have invalid operands or may have other results that need promotion, we just
1277/// know that (at least) one result needs expansion.
1278void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1279 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1280 SDValue Lo, Hi;
1281 Lo = Hi = SDValue();
1282
1283 // See if the target wants to custom expand this node.
1284 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1285 return;
1286
1287 switch (N->getOpcode()) {
1288 default:
1289#ifndef NDEBUG
1290 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1291 N->dump(&DAG); dbgs() << "\n";
1292#endif
1293 report_fatal_error("Do not know how to expand the result of this "
1294 "operator!");
1295
1296 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1297 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1298 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1299
1300 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1301 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1302 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1303 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1304 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1305 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1306
1307 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1308 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1309 case ISD::STRICT_FMINNUM:
1310 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1311 case ISD::STRICT_FMAXNUM:
1312 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1313 case ISD::STRICT_FADD:
1314 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1315 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1316 case ISD::STRICT_FCEIL:
1317 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1318 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1319 case ISD::STRICT_FCOS:
1320 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1321 case ISD::STRICT_FDIV:
1322 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1323 case ISD::STRICT_FEXP:
1324 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1325 case ISD::STRICT_FEXP2:
1326 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1327 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1328 case ISD::STRICT_FFLOOR:
1329 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1330 case ISD::STRICT_FLOG:
1331 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1332 case ISD::STRICT_FLOG2:
1333 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1334 case ISD::STRICT_FLOG10:
1335 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1336 case ISD::STRICT_FMA:
1337 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1338 case ISD::STRICT_FMUL:
1339 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1340 case ISD::STRICT_FNEARBYINT:
1341 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1342 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1343 case ISD::STRICT_FP_EXTEND:
1344 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1345 case ISD::STRICT_FPOW:
1346 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1347 case ISD::STRICT_FPOWI:
1348 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1349 case ISD::FLDEXP:
1350 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1351 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1352 case ISD::STRICT_FRINT:
1353 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1354 case ISD::STRICT_FROUND:
1355 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1356 case ISD::STRICT_FROUNDEVEN:
1357 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1358 case ISD::STRICT_FSIN:
1359 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1360 case ISD::STRICT_FSQRT:
1361 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1362 case ISD::STRICT_FSUB:
1363 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1364 case ISD::STRICT_FTRUNC:
1365 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1366 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1367 case ISD::STRICT_SINT_TO_FP:
1368 case ISD::STRICT_UINT_TO_FP:
1369 case ISD::SINT_TO_FP:
1370 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1371 case ISD::STRICT_FREM:
1372 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1373 }
1374
1375 // If Lo/Hi is null, the sub-method took care of registering results etc.
1376 if (Lo.getNode())
1377 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1378}
1379
1380void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1381 SDValue &Hi) {
1382 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1383 assert(NVT.getSizeInBits() == 64 &&
1384 "Do not know how to expand this float constant!");
1385 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1386 SDLoc dl(N);
1387 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1388 APInt(64, C.getRawData()[1])),
1389 dl, NVT);
1390 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1391 APInt(64, C.getRawData()[0])),
1392 dl, NVT);
1393}
1394
1395void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1396 SDValue &Lo, SDValue &Hi) {
1397 bool IsStrict = N->isStrictFPOpcode();
1398 unsigned Offset = IsStrict ? 1 : 0;
1399 SDValue Op = N->getOperand(0 + Offset);
1400 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1401 TargetLowering::MakeLibCallOptions CallOptions;
1402 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1403 Op, CallOptions, SDLoc(N),
1404 Chain);
1405 if (IsStrict)
1406 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1407 GetPairElements(Tmp.first, Lo, Hi);
1408}
1409
1410void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1411 SDValue &Lo, SDValue &Hi) {
1412 bool IsStrict = N->isStrictFPOpcode();
1413 unsigned Offset = IsStrict ? 1 : 0;
1414 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1415 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1416 TargetLowering::MakeLibCallOptions CallOptions;
1417 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1418 Ops, CallOptions, SDLoc(N),
1419 Chain);
1420 if (IsStrict)
1421 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1422 GetPairElements(Tmp.first, Lo, Hi);
1423}
1424
1425void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1426 SDValue &Hi) {
1427 assert(N->getValueType(0) == MVT::ppcf128 &&
1428 "Logic only correct for ppcf128!");
1429 SDLoc dl(N);
1430 SDValue Tmp;
1431 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1432 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1433 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1434 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1435 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1436 ISD::SETEQ);
1437}
1438
1439void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1440 SDValue &Hi) {
1441 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1442 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1443 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1444 RTLIB::FMIN_PPCF128), Lo, Hi);
1445}
1446
1447void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1448 SDValue &Hi) {
1449 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1450 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1451 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1452 RTLIB::FMAX_PPCF128), Lo, Hi);
1453}
1454
1455void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1456 SDValue &Hi) {
1457 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1458 RTLIB::ADD_F32, RTLIB::ADD_F64,
1459 RTLIB::ADD_F80, RTLIB::ADD_F128,
1460 RTLIB::ADD_PPCF128), Lo, Hi);
1461}
1462
1463void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1464 SDValue &Hi) {
1465 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1466 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1467 RTLIB::CBRT_F128,
1468 RTLIB::CBRT_PPCF128), Lo, Hi);
1469}
1470
1471void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1472 SDValue &Lo, SDValue &Hi) {
1473 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1474 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1475 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1476 RTLIB::CEIL_PPCF128), Lo, Hi);
1477}
1478
1479void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1480 SDValue &Lo, SDValue &Hi) {
1481 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1482 RTLIB::COPYSIGN_F32,
1483 RTLIB::COPYSIGN_F64,
1484 RTLIB::COPYSIGN_F80,
1485 RTLIB::COPYSIGN_F128,
1486 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1487}
1488
1489void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1490 SDValue &Lo, SDValue &Hi) {
1491 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1492 RTLIB::COS_F32, RTLIB::COS_F64,
1493 RTLIB::COS_F80, RTLIB::COS_F128,
1494 RTLIB::COS_PPCF128), Lo, Hi);
1495}
1496
1497void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1498 SDValue &Hi) {
1499 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1500 RTLIB::DIV_F32,
1501 RTLIB::DIV_F64,
1502 RTLIB::DIV_F80,
1503 RTLIB::DIV_F128,
1504 RTLIB::DIV_PPCF128), Lo, Hi);
1505}
1506
1507void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1508 SDValue &Lo, SDValue &Hi) {
1509 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1510 RTLIB::EXP_F32, RTLIB::EXP_F64,
1511 RTLIB::EXP_F80, RTLIB::EXP_F128,
1512 RTLIB::EXP_PPCF128), Lo, Hi);
1513}
1514
1515void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1516 SDValue &Lo, SDValue &Hi) {
1517 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1518 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1519 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1520 RTLIB::EXP2_PPCF128), Lo, Hi);
1521}
1522
1523void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1524 SDValue &Hi) {
1525 ExpandFloatRes_Unary(N,
1526 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1527 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1528 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1529 Lo, Hi);
1530}
1531
1532void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1533 SDValue &Lo, SDValue &Hi) {
1534 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1535 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1536 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1537 RTLIB::FLOOR_PPCF128), Lo, Hi);
1538}
1539
1540void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1541 SDValue &Lo, SDValue &Hi) {
1542 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1543 RTLIB::LOG_F32, RTLIB::LOG_F64,
1544 RTLIB::LOG_F80, RTLIB::LOG_F128,
1545 RTLIB::LOG_PPCF128), Lo, Hi);
1546}
1547
1548void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1549 SDValue &Lo, SDValue &Hi) {
1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1551 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1552 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1553 RTLIB::LOG2_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1560 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1561 RTLIB::LOG10_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 bool IsStrict = N->isStrictFPOpcode();
1567 unsigned Offset = IsStrict ? 1 : 0;
1568 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1569 N->getOperand(2 + Offset) };
1570 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1571 TargetLowering::MakeLibCallOptions CallOptions;
1572 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1573 RTLIB::FMA_F32,
1574 RTLIB::FMA_F64,
1575 RTLIB::FMA_F80,
1576 RTLIB::FMA_F128,
1577 RTLIB::FMA_PPCF128),
1578 N->getValueType(0), Ops, CallOptions,
1579 SDLoc(N), Chain);
1580 if (IsStrict)
1581 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1582 GetPairElements(Tmp.first, Lo, Hi);
1583}
1584
1585void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1586 SDValue &Hi) {
1587 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1588 RTLIB::MUL_F32,
1589 RTLIB::MUL_F64,
1590 RTLIB::MUL_F80,
1591 RTLIB::MUL_F128,
1592 RTLIB::MUL_PPCF128), Lo, Hi);
1593}
1594
1595void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1596 SDValue &Lo, SDValue &Hi) {
1597 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1598 RTLIB::NEARBYINT_F32,
1599 RTLIB::NEARBYINT_F64,
1600 RTLIB::NEARBYINT_F80,
1601 RTLIB::NEARBYINT_F128,
1602 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1603}
1604
1605void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1606 SDValue &Hi) {
1607 SDLoc dl(N);
1608 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1609 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1610 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1611}
1612
1613void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1614 SDValue &Hi) {
1615 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1616 SDLoc dl(N);
1617 bool IsStrict = N->isStrictFPOpcode();
1618
1619 SDValue Chain;
1620 if (IsStrict) {
1621 // If the expanded type is the same as the input type, just bypass the node.
1622 if (NVT == N->getOperand(1).getValueType()) {
1623 Hi = N->getOperand(1);
1624 Chain = N->getOperand(0);
1625 } else {
1626 // Other we need to extend.
1627 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1628 { N->getOperand(0), N->getOperand(1) });
1629 Chain = Hi.getValue(1);
1630 }
1631 } else {
1632 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1633 }
1634
1635 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1636 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1637
1638 if (IsStrict)
1639 ReplaceValueWith(SDValue(N, 1), Chain);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1643 SDValue &Lo, SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1645 RTLIB::POW_F32, RTLIB::POW_F64,
1646 RTLIB::POW_F80, RTLIB::POW_F128,
1647 RTLIB::POW_PPCF128), Lo, Hi);
1648}
1649
1650void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1651 SDValue &Lo, SDValue &Hi) {
1652 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1653}
1654
1655void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1656 SDValue &Hi) {
1657 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1658}
1659
1660void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 assert(N->getValueType(0) == MVT::ppcf128 &&
1663 "Logic only correct for ppcf128!");
1664
1665 SDLoc dl(N);
1666 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1667 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1668 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1669}
1670
1671void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1672 SDValue &Lo, SDValue &Hi) {
1673 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1674 RTLIB::REM_F32, RTLIB::REM_F64,
1675 RTLIB::REM_F80, RTLIB::REM_F128,
1676 RTLIB::REM_PPCF128), Lo, Hi);
1677}
1678
1679void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1680 SDValue &Lo, SDValue &Hi) {
1681 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1682 RTLIB::RINT_F32, RTLIB::RINT_F64,
1683 RTLIB::RINT_F80, RTLIB::RINT_F128,
1684 RTLIB::RINT_PPCF128), Lo, Hi);
1685}
1686
1687void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1688 SDValue &Lo, SDValue &Hi) {
1689 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1690 RTLIB::ROUND_F32,
1691 RTLIB::ROUND_F64,
1692 RTLIB::ROUND_F80,
1693 RTLIB::ROUND_F128,
1694 RTLIB::ROUND_PPCF128), Lo, Hi);
1695}
1696
1697void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1698 SDValue &Lo, SDValue &Hi) {
1699 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1700 RTLIB::ROUNDEVEN_F32,
1701 RTLIB::ROUNDEVEN_F64,
1702 RTLIB::ROUNDEVEN_F80,
1703 RTLIB::ROUNDEVEN_F128,
1704 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1705}
1706
1707void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1708 SDValue &Lo, SDValue &Hi) {
1709 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1710 RTLIB::SIN_F32, RTLIB::SIN_F64,
1711 RTLIB::SIN_F80, RTLIB::SIN_F128,
1712 RTLIB::SIN_PPCF128), Lo, Hi);
1713}
1714
1715void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1716 SDValue &Lo, SDValue &Hi) {
1717 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1718 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1719 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1720 RTLIB::SQRT_PPCF128), Lo, Hi);
1721}
1722
1723void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1724 SDValue &Hi) {
1725 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1726 RTLIB::SUB_F32,
1727 RTLIB::SUB_F64,
1728 RTLIB::SUB_F80,
1729 RTLIB::SUB_F128,
1730 RTLIB::SUB_PPCF128), Lo, Hi);
1731}
1732
1733void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1734 SDValue &Lo, SDValue &Hi) {
1735 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1736 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1737 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1738 RTLIB::TRUNC_PPCF128), Lo, Hi);
1739}
1740
1741void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1743 if (ISD::isNormalLoad(N)) {
1744 ExpandRes_NormalLoad(N, Lo, Hi);
1745 return;
1746 }
1747
1748 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1749 LoadSDNode *LD = cast<LoadSDNode>(N);
1750 SDValue Chain = LD->getChain();
1751 SDValue Ptr = LD->getBasePtr();
1752 SDLoc dl(N);
1753
1754 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1755 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1756 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1757
1758 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1759 LD->getMemoryVT(), LD->getMemOperand());
1760
1761 // Remember the chain.
1762 Chain = Hi.getValue(1);
1763
1764 // The low part is zero.
1765 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1766 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1767
1768 // Modified the chain - switch anything that used the old chain to use the
1769 // new one.
1770 ReplaceValueWith(SDValue(LD, 1), Chain);
1771}
1772
1773void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1774 SDValue &Hi) {
1775 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1776 EVT VT = N->getValueType(0);
1777 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1778 bool Strict = N->isStrictFPOpcode();
1779 SDValue Src = N->getOperand(Strict ? 1 : 0);
1780 EVT SrcVT = Src.getValueType();
1781 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1782 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1783 SDLoc dl(N);
1784 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1785
1786 // TODO: Any other flags to propagate?
1787 SDNodeFlags Flags;
1788 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1789
1790 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1791 // When promoting partial word types to i32 we must honor the signedness,
1792 // though.
1793 if (SrcVT.bitsLE(MVT::i32)) {
1794 // The integer can be represented exactly in an f64.
1795 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1796 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1797 if (Strict) {
1798 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1799 {Chain, Src}, Flags);
1800 Chain = Hi.getValue(1);
1801 } else
1802 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1803 } else {
1804 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1805 if (SrcVT.bitsLE(MVT::i64)) {
1806 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1807 MVT::i64, Src);
1808 LC = RTLIB::SINTTOFP_I64_PPCF128;
1809 } else if (SrcVT.bitsLE(MVT::i128)) {
1810 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1811 LC = RTLIB::SINTTOFP_I128_PPCF128;
1812 }
1813 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1814
1815 TargetLowering::MakeLibCallOptions CallOptions;
1816 CallOptions.setSExt(true);
1817 std::pair<SDValue, SDValue> Tmp =
1818 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1819 if (Strict)
1820 Chain = Tmp.second;
1821 GetPairElements(Tmp.first, Lo, Hi);
1822 }
1823
1824 // No need to complement for unsigned 32-bit integers
1825 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1826 if (Strict)
1827 ReplaceValueWith(SDValue(N, 1), Chain);
1828
1829 return;
1830 }
1831
1832 // Unsigned - fix up the SINT_TO_FP value just calculated.
1833 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1834 // keep semantics correctness if the integer is not exactly representable
1835 // here. See ExpandLegalINT_TO_FP.
1836 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1837 SrcVT = Src.getValueType();
1838
1839 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1840 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1841 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1842 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1843 ArrayRef<uint64_t> Parts;
1844
1845 switch (SrcVT.getSimpleVT().SimpleTy) {
1846 default:
1847 llvm_unreachable("Unsupported UINT_TO_FP!");
1848 case MVT::i32:
1849 Parts = TwoE32;
1850 break;
1851 case MVT::i64:
1852 Parts = TwoE64;
1853 break;
1854 case MVT::i128:
1855 Parts = TwoE128;
1856 break;
1857 }
1858
1859 // TODO: Are there other fast-math-flags to propagate to this FADD?
1860 SDValue NewLo = DAG.getConstantFP(
1861 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1862 if (Strict) {
1863 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1864 {Chain, Hi, NewLo}, Flags);
1865 Chain = Lo.getValue(1);
1866 ReplaceValueWith(SDValue(N, 1), Chain);
1867 } else
1868 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1869 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1870 Lo, Hi, ISD::SETLT);
1871 GetPairElements(Lo, Lo, Hi);
1872}
1873
1874
1875//===----------------------------------------------------------------------===//
1876// Float Operand Expansion
1877//===----------------------------------------------------------------------===//
1878
1879/// ExpandFloatOperand - This method is called when the specified operand of the
1880/// specified node is found to need expansion. At this point, all of the result
1881/// types of the node are known to be legal, but other operands of the node may
1882/// need promotion or expansion as well as the specified one.
1883bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1884 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1885 SDValue Res = SDValue();
1886
1887 // See if the target wants to custom expand this node.
1888 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1889 return false;
1890
1891 switch (N->getOpcode()) {
1892 default:
1893#ifndef NDEBUG
1894 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1895 N->dump(&DAG); dbgs() << "\n";
1896#endif
1897 report_fatal_error("Do not know how to expand this operator's operand!");
1898
1899 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1900 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1901 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1902
1903 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1904 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1905 case ISD::STRICT_FP_ROUND:
1906 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1907 case ISD::STRICT_FP_TO_SINT:
1908 case ISD::STRICT_FP_TO_UINT:
1909 case ISD::FP_TO_SINT:
1910 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1911 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1912 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1913 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1914 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1915 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1916 case ISD::STRICT_FSETCC:
1917 case ISD::STRICT_FSETCCS:
1918 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1919 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1920 OpNo); break;
1921 }
1922
1923 // If the result is null, the sub-method took care of registering results etc.
1924 if (!Res.getNode()) return false;
1925
1926 // If the result is N, the sub-method updated N in place. Tell the legalizer
1927 // core about this.
1928 if (Res.getNode() == N)
1929 return true;
1930
1931 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1932 "Invalid operand expansion");
1933
1934 ReplaceValueWith(SDValue(N, 0), Res);
1935 return false;
1936}
1937
1938/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1939/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1940void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1941 SDValue &NewRHS,
1942 ISD::CondCode &CCCode,
1943 const SDLoc &dl, SDValue &Chain,
1944 bool IsSignaling) {
1945 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1946 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1947 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1948
1949 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1950
1951 // FIXME: This generated code sucks. We want to generate
1952 // FCMPU crN, hi1, hi2
1953 // BNE crN, L:
1954 // FCMPU crN, lo1, lo2
1955 // The following can be improved, but not that much.
1956 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1957 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1958 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1959 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1960 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1961 RHSLo, CCCode, OutputChain, IsSignaling);
1962 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1963 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1964 Tmp1 =
1965 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1966 ISD::SETUNE, OutputChain, IsSignaling);
1967 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1968 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1969 RHSHi, CCCode, OutputChain, IsSignaling);
1970 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1971 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1972 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1973 NewRHS = SDValue(); // LHS is the result, not a compare.
1974 Chain = OutputChain;
1975}
1976
1977SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1978 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1979 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1980 SDValue Chain;
1981 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1982
1983 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1984 // against zero to select between true and false values.
1985 if (!NewRHS.getNode()) {
1986 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1987 CCCode = ISD::SETNE;
1988 }
1989
1990 // Update N to have the operands specified.
1991 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1992 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1993 N->getOperand(4)), 0);
1994}
1995
1996SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1997 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1998 "Logic only correct for ppcf128!");
1999 SDValue Lo, Hi;
2000 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2001 // The ppcf128 value is providing only the sign; take it from the
2002 // higher-order double (which must have the larger magnitude).
2003 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2004 N->getValueType(0), N->getOperand(0), Hi);
2005}
2006
2007SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2008 bool IsStrict = N->isStrictFPOpcode();
2009 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2010 "Logic only correct for ppcf128!");
2011 SDValue Lo, Hi;
2012 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2013
2014 if (!IsStrict)
2015 // Round it the rest of the way (e.g. to f32) if needed.
2016 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2017 N->getValueType(0), Hi, N->getOperand(1));
2018
2019 // Eliminate the node if the input float type is the same as the output float
2020 // type.
2021 if (Hi.getValueType() == N->getValueType(0)) {
2022 // Connect the output chain to the input chain, unlinking the node.
2023 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2024 ReplaceValueWith(SDValue(N, 0), Hi);
2025 return SDValue();
2026 }
2027
2028 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2029 {N->getValueType(0), MVT::Other},
2030 {N->getOperand(0), Hi, N->getOperand(2)});
2031 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2032 ReplaceValueWith(SDValue(N, 0), Expansion);
2033 return SDValue();
2034}
2035
2036SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2037 EVT RVT = N->getValueType(0);
2038 SDLoc dl(N);
2039
2040 bool IsStrict = N->isStrictFPOpcode();
2041 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2042 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2043 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2044 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2045
2046 EVT NVT;
2047 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2048 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2049 "Unsupported FP_TO_XINT!");
2050 TargetLowering::MakeLibCallOptions CallOptions;
2051 std::pair<SDValue, SDValue> Tmp =
2052 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2053 if (!IsStrict)
2054 return Tmp.first;
2055
2056 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2057 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2058 return SDValue();
2059}
2060
2061SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2062 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2063 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2064 SDValue Chain;
2065 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2066
2067 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2068 // against zero to select between true and false values.
2069 if (!NewRHS.getNode()) {
2070 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2071 CCCode = ISD::SETNE;
2072 }
2073
2074 // Update N to have the operands specified.
2075 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2076 N->getOperand(2), N->getOperand(3),
2077 DAG.getCondCode(CCCode)), 0);
2078}
2079
2080SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2081 bool IsStrict = N->isStrictFPOpcode();
2082 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2083 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2084 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2085 ISD::CondCode CCCode =
2086 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2087 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2088 N->getOpcode() == ISD::STRICT_FSETCCS);
2089
2090 // FloatExpandSetCCOperands always returned a scalar.
2091 assert(!NewRHS.getNode() && "Expect to return scalar");
2092 assert(NewLHS.getValueType() == N->getValueType(0) &&
2093 "Unexpected setcc expansion!");
2094 if (Chain) {
2095 ReplaceValueWith(SDValue(N, 0), NewLHS);
2096 ReplaceValueWith(SDValue(N, 1), Chain);
2097 return SDValue();
2098 }
2099 return NewLHS;
2100}
2101
2102SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2103 if (ISD::isNormalStore(N))
2104 return ExpandOp_NormalStore(N, OpNo);
2105
2106 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2107 assert(OpNo == 1 && "Can only expand the stored value so far");
2108 StoreSDNode *ST = cast<StoreSDNode>(N);
2109
2110 SDValue Chain = ST->getChain();
2111 SDValue Ptr = ST->getBasePtr();
2112
2113 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2114 ST->getValue().getValueType());
2115 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2116 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2117 (void)NVT;
2118
2119 SDValue Lo, Hi;
2120 GetExpandedOp(ST->getValue(), Lo, Hi);
2121
2122 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2123 ST->getMemoryVT(), ST->getMemOperand());
2124}
2125
2126SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2127 EVT RVT = N->getValueType(0);
2128 EVT RetVT = N->getOperand(0).getValueType();
2129 TargetLowering::MakeLibCallOptions CallOptions;
2130 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2131 RTLIB::LROUND_F32,
2132 RTLIB::LROUND_F64,
2133 RTLIB::LROUND_F80,
2134 RTLIB::LROUND_F128,
2135 RTLIB::LROUND_PPCF128),
2136 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2137}
2138
2139SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2140 EVT RVT = N->getValueType(0);
2141 EVT RetVT = N->getOperand(0).getValueType();
2142 TargetLowering::MakeLibCallOptions CallOptions;
2143 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2144 RTLIB::LLROUND_F32,
2145 RTLIB::LLROUND_F64,
2146 RTLIB::LLROUND_F80,
2147 RTLIB::LLROUND_F128,
2148 RTLIB::LLROUND_PPCF128),
2149 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2150}
2151
2152SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2153 EVT RVT = N->getValueType(0);
2154 EVT RetVT = N->getOperand(0).getValueType();
2155 TargetLowering::MakeLibCallOptions CallOptions;
2156 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2157 RTLIB::LRINT_F32,
2158 RTLIB::LRINT_F64,
2159 RTLIB::LRINT_F80,
2160 RTLIB::LRINT_F128,
2161 RTLIB::LRINT_PPCF128),
2162 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2163}
2164
2165SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2166 EVT RVT = N->getValueType(0);
2167 EVT RetVT = N->getOperand(0).getValueType();
2168 TargetLowering::MakeLibCallOptions CallOptions;
2169 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2170 RTLIB::LLRINT_F32,
2171 RTLIB::LLRINT_F64,
2172 RTLIB::LLRINT_F80,
2173 RTLIB::LLRINT_F128,
2174 RTLIB::LLRINT_PPCF128),
2175 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2176}
2177
2178//===----------------------------------------------------------------------===//
2179// Float Operand Promotion
2180//===----------------------------------------------------------------------===//
2181//
2182
2183static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2184 if (OpVT == MVT::f16) {
2185 return ISD::FP16_TO_FP;
2186 } else if (RetVT == MVT::f16) {
2187 return ISD::FP_TO_FP16;
2188 } else if (OpVT == MVT::bf16) {
2189 return ISD::BF16_TO_FP;
2190 } else if (RetVT == MVT::bf16) {
2191 return ISD::FP_TO_BF16;
2192 }
2193
2194 report_fatal_error("Attempt at an invalid promotion-related conversion");
2195}
2196
2197static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2198 if (OpVT == MVT::f16)
2199 return ISD::STRICT_FP16_TO_FP;
2200
2201 if (RetVT == MVT::f16)
2202 return ISD::STRICT_FP_TO_FP16;
2203
2204 if (OpVT == MVT::bf16)
2205 return ISD::STRICT_BF16_TO_FP;
2206
2207 if (RetVT == MVT::bf16)
2208 return ISD::STRICT_FP_TO_BF16;
2209
2210 report_fatal_error("Attempt at an invalid promotion-related conversion");
2211}
2212
2213bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2214 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2215 SDValue R = SDValue();
2216
2217 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2218 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2219 return false;
2220 }
2221
2222 // Nodes that use a promotion-requiring floating point operand, but doesn't
2223 // produce a promotion-requiring floating point result, need to be legalized
2224 // to use the promoted float operand. Nodes that produce at least one
2225 // promotion-requiring floating point result have their operands legalized as
2226 // a part of PromoteFloatResult.
2227 // clang-format off
2228 switch (N->getOpcode()) {
2229 default:
2230 #ifndef NDEBUG
2231 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2232 N->dump(&DAG); dbgs() << "\n";
2233 #endif
2234 report_fatal_error("Do not know how to promote this operator's operand!");
2235
2236 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2237 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2238 case ISD::FP_TO_SINT:
2239 case ISD::FP_TO_UINT:
2240 case ISD::LRINT:
2241 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2242 case ISD::FP_TO_SINT_SAT:
2243 case ISD::FP_TO_UINT_SAT:
2244 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2245 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2246 case ISD::STRICT_FP_EXTEND:
2247 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2248 break;
2249 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2250 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2251 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2252 }
2253 // clang-format on
2254
2255 if (R.getNode())
2256 ReplaceValueWith(SDValue(N, 0), R);
2257 return false;
2258}
2259
2260SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2261 SDValue Op = N->getOperand(0);
2262 EVT OpVT = Op->getValueType(0);
2263
2264 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2265 EVT PromotedVT = Promoted->getValueType(0);
2266
2267 // Convert the promoted float value to the desired IVT.
2268 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2269 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2270 IVT, Promoted);
2271 // The final result type might not be an scalar so we need a bitcast. The
2272 // bitcast will be further legalized if needed.
2273 return DAG.getBitcast(N->getValueType(0), Convert);
2274}
2275
2276// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2277// PromoteFloatRes_FCOPYSIGN.
2278SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2279 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2280 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2281
2282 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2283 N->getOperand(0), Op1);
2284}
2285
2286// Convert the promoted float value to the desired integer type
2287SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2288 SDValue Op = GetPromotedFloat(N->getOperand(0));
2289 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2290}
2291
2292SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2293 unsigned OpNo) {
2294 SDValue Op = GetPromotedFloat(N->getOperand(0));
2295 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2296 N->getOperand(1));
2297}
2298
2299SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2300 SDValue Op = GetPromotedFloat(N->getOperand(0));
2301 EVT VT = N->getValueType(0);
2302
2303 // Desired VT is same as promoted type. Use promoted float directly.
2304 if (VT == Op->getValueType(0))
2305 return Op;
2306
2307 // Else, extend the promoted float value to the desired VT.
2308 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2309}
2310
2311SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2312 unsigned OpNo) {
2313 assert(OpNo == 1 && "Promoting unpromotable operand");
2314
2315 SDValue Op = GetPromotedFloat(N->getOperand(1));
2316 EVT VT = N->getValueType(0);
2317
2318 // Desired VT is same as promoted type. Use promoted float directly.
2319 if (VT == Op->getValueType(0)) {
2320 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2321 return Op;
2322 }
2323
2324 // Else, extend the promoted float value to the desired VT.
2325 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2326 N->getOperand(0), Op);
2327 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2328 return Res;
2329}
2330
2331// Promote the float operands used for comparison. The true- and false-
2332// operands have the same type as the result and are promoted, if needed, by
2333// PromoteFloatRes_SELECT_CC
2334SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2335 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2336 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2337
2338 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2339 LHS, RHS, N->getOperand(2), N->getOperand(3),
2340 N->getOperand(4));
2341}
2342
2343// Construct a SETCC that compares the promoted values and sets the conditional
2344// code.
2345SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2346 EVT VT = N->getValueType(0);
2347 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2348 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2349 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2350
2351 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2352
2353}
2354
2355// Lower the promoted Float down to the integer value of same size and construct
2356// a STORE of the integer value.
2357SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2358 StoreSDNode *ST = cast<StoreSDNode>(N);
2359 SDValue Val = ST->getValue();
2360 SDLoc DL(N);
2361
2362 SDValue Promoted = GetPromotedFloat(Val);
2363 EVT VT = ST->getOperand(1).getValueType();
2364 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2365
2366 SDValue NewVal;
2367 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2368 IVT, Promoted);
2369
2370 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2371 ST->getMemOperand());
2372}
2373
2374//===----------------------------------------------------------------------===//
2375// Float Result Promotion
2376//===----------------------------------------------------------------------===//
2377
2378void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2379 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2380 SDValue R = SDValue();
2381
2382 // See if the target wants to custom expand this node.
2383 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2384 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2385 return;
2386 }
2387
2388 switch (N->getOpcode()) {
2389 // These opcodes cannot appear if promotion of FP16 is done in the backend
2390 // instead of Clang
2391 case ISD::FP16_TO_FP:
2392 case ISD::FP_TO_FP16:
2393 default:
2394#ifndef NDEBUG
2395 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2396 N->dump(&DAG); dbgs() << "\n";
2397#endif
2398 report_fatal_error("Do not know how to promote this operator's result!");
2399
2400 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2401 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2402 case ISD::EXTRACT_VECTOR_ELT:
2403 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2404 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2405
2406 // Unary FP Operations
2407 case ISD::FABS:
2408 case ISD::FCBRT:
2409 case ISD::FCEIL:
2410 case ISD::FCOS:
2411 case ISD::FEXP:
2412 case ISD::FEXP2:
2413 case ISD::FEXP10:
2414 case ISD::FFLOOR:
2415 case ISD::FLOG:
2416 case ISD::FLOG2:
2417 case ISD::FLOG10:
2418 case ISD::FNEARBYINT:
2419 case ISD::FNEG:
2420 case ISD::FRINT:
2421 case ISD::FROUND:
2422 case ISD::FROUNDEVEN:
2423 case ISD::FSIN:
2424 case ISD::FSQRT:
2425 case ISD::FTRUNC:
2426 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2427
2428 // Binary FP Operations
2429 case ISD::FADD:
2430 case ISD::FDIV:
2431 case ISD::FMAXIMUM:
2432 case ISD::FMINIMUM:
2433 case ISD::FMAXNUM:
2434 case ISD::FMINNUM:
2435 case ISD::FMUL:
2436 case ISD::FPOW:
2437 case ISD::FREM:
2438 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2439
2440 case ISD::FMA: // FMA is same as FMAD
2441 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2442
2443 case ISD::FPOWI:
2444 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2445 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2446
2447 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2448 case ISD::STRICT_FP_ROUND:
2449 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2450 break;
2451 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2452 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2453 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2454
2455 case ISD::SINT_TO_FP:
2456 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2457 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2458 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2459 case ISD::VECREDUCE_FADD:
2460 case ISD::VECREDUCE_FMUL:
2461 case ISD::VECREDUCE_FMIN:
2462 case ISD::VECREDUCE_FMAX:
2463 case ISD::VECREDUCE_FMAXIMUM:
2464 case ISD::VECREDUCE_FMINIMUM:
2465 R = PromoteFloatRes_VECREDUCE(N);
2466 break;
2467 case ISD::VECREDUCE_SEQ_FADD:
2468 case ISD::VECREDUCE_SEQ_FMUL:
2469 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2470 break;
2471 }
2472
2473 if (R.getNode())
2474 SetPromotedFloat(SDValue(N, ResNo), R);
2475}
2476
2477// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2478// At this point, it is not possible to determine if the bitcast value is
2479// eventually stored to memory or promoted to f32 or promoted to a floating
2480// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2481// STORE promotion handlers.
2482SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2483 EVT VT = N->getValueType(0);
2484 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2485 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2486 // bitcast will be legalized further if necessary.
2487 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2488 N->getOperand(0).getValueType().getSizeInBits());
2489 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2490 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2491}
2492
2493SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2494 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2495 EVT VT = N->getValueType(0);
2496 SDLoc DL(N);
2497
2498 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2499 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2500 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2501 IVT);
2502
2503 // Convert the Constant to the desired FP type
2504 // FIXME We might be able to do the conversion during compilation and get rid
2505 // of it from the object code
2506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2507 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2508}
2509
2510// If the Index operand is a constant, try to redirect the extract operation to
2511// the correct legalized vector. If not, bit-convert the input vector to
2512// equivalent integer vector. Extract the element as an (bit-cast) integer
2513// value and convert it to the promoted type.
2514SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2515 SDLoc DL(N);
2516
2517 // If the index is constant, try to extract the value from the legalized
2518 // vector type.
2519 if (isa<ConstantSDNode>(N->getOperand(1))) {
2520 SDValue Vec = N->getOperand(0);
2521 SDValue Idx = N->getOperand(1);
2522 EVT VecVT = Vec->getValueType(0);
2523 EVT EltVT = VecVT.getVectorElementType();
2524
2525 uint64_t IdxVal = Idx->getAsZExtVal();
2526
2527 switch (getTypeAction(VecVT)) {
2528 default: break;
2529 case TargetLowering::TypeScalarizeVector: {
2530 SDValue Res = GetScalarizedVector(N->getOperand(0));
2531 ReplaceValueWith(SDValue(N, 0), Res);
2532 return SDValue();
2533 }
2534 case TargetLowering::TypeWidenVector: {
2535 Vec = GetWidenedVector(Vec);
2536 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2537 ReplaceValueWith(SDValue(N, 0), Res);
2538 return SDValue();
2539 }
2540 case TargetLowering::TypeSplitVector: {
2541 SDValue Lo, Hi;
2542 GetSplitVector(Vec, Lo, Hi);
2543
2544 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2545 SDValue Res;
2546 if (IdxVal < LoElts)
2547 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2548 else
2549 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2550 DAG.getConstant(IdxVal - LoElts, DL,
2551 Idx.getValueType()));
2552 ReplaceValueWith(SDValue(N, 0), Res);
2553 return SDValue();
2554 }
2555
2556 }
2557 }
2558
2559 // Bit-convert the input vector to the equivalent integer vector
2560 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2561 EVT IVT = NewOp.getValueType().getVectorElementType();
2562
2563 // Extract the element as an (bit-cast) integer value
2564 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2565 NewOp, N->getOperand(1));
2566
2567 // Convert the element to the desired FP type
2568 EVT VT = N->getValueType(0);
2569 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2570 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2571}
2572
2573// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2574// needs promotion, so does the argument X. Note that Y, if needed, will be
2575// handled during operand promotion.
2576SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2577 EVT VT = N->getValueType(0);
2578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2579 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2580
2581 SDValue Op1 = N->getOperand(1);
2582
2583 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2584}
2585
2586// Unary operation where the result and the operand have PromoteFloat type
2587// action. Construct a new SDNode with the promoted float value of the old
2588// operand.
2589SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2590 EVT VT = N->getValueType(0);
2591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2592 SDValue Op = GetPromotedFloat(N->getOperand(0));
2593
2594 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2595}
2596
2597// Binary operations where the result and both operands have PromoteFloat type
2598// action. Construct a new SDNode with the promoted float values of the old
2599// operands.
2600SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2601 EVT VT = N->getValueType(0);
2602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2603 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2604 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2605 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2606}
2607
2608SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2609 EVT VT = N->getValueType(0);
2610 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2611 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2612 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2613 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2614
2615 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2616}
2617
2618// Promote the Float (first) operand and retain the Integer (second) operand
2619SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2620 EVT VT = N->getValueType(0);
2621 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2622 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2623 SDValue Op1 = N->getOperand(1);
2624
2625 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2626}
2627
2628SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2629 EVT VT = N->getValueType(0);
2630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2631 SDValue Op = GetPromotedFloat(N->getOperand(0));
2632 SDValue Res =
2633 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2634
2635 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2636 return Res;
2637}
2638
2639// Explicit operation to reduce precision. Reduce the value to half precision
2640// and promote it back to the legal type.
2641SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2642 SDLoc DL(N);
2643
2644 SDValue Op = N->getOperand(0);
2645 EVT VT = N->getValueType(0);
2646 EVT OpVT = Op->getValueType(0);
2647 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2648 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2649
2650 // Round promoted float to desired precision
2651 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2652 // Promote it back to the legal output type
2653 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2654}
2655
2656// Explicit operation to reduce precision. Reduce the value to half precision
2657// and promote it back to the legal type.
2658SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2659 SDLoc DL(N);
2660
2661 SDValue Chain = N->getOperand(0);
2662 SDValue Op = N->getOperand(1);
2663 EVT VT = N->getValueType(0);
2664 EVT OpVT = Op->getValueType(0);
2665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2666 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2667
2668 // Round promoted float to desired precision
2669 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2670 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2671 // Promote it back to the legal output type
2672 SDValue Res =
2673 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2674 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2675 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2676 return Res;
2677}
2678
2679SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2680 LoadSDNode *L = cast<LoadSDNode>(N);
2681 EVT VT = N->getValueType(0);
2682
2683 // Load the value as an integer value with the same number of bits.
2684 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2685 SDValue newL = DAG.getLoad(
2686 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2687 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2688 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2689 // Legalize the chain result by replacing uses of the old value chain with the
2690 // new one
2691 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2692
2693 // Convert the integer value to the desired FP type
2694 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2695 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2696}
2697
2698// Construct a new SELECT node with the promoted true- and false- values.
2699SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2700 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2701 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2702
2703 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2704 N->getOperand(0), TrueVal, FalseVal);
2705}
2706
2707// Construct a new SELECT_CC node with the promoted true- and false- values.
2708// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2709SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2710 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2711 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2712
2713 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2714 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2715 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2716}
2717
2718// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2719// float type.
2720SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2721 SDLoc DL(N);
2722 EVT VT = N->getValueType(0);
2723 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2724 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2725 // Round the value to the desired precision (that of the source type).
2726 return DAG.getNode(
2727 ISD::FP_EXTEND, DL, NVT,
2728 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2729 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2730}
2731
2732SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2733 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2734 N->getValueType(0)));
2735}
2736
2737SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2738 // Expand and promote recursively.
2739 // TODO: This is non-optimal, but dealing with the concurrently happening
2740 // vector-legalization is non-trivial. We could do something similar to
2741 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2742 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2743 return SDValue();
2744}
2745
2746SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2747 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2748 return SDValue();
2749}
2750
2751SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2752 EVT VT = N->getValueType(0);
2753
2754 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2755 SDLoc SL(N);
2756
2757 SDValue CastVal = BitConvertToInteger(AM->getVal());
2758 EVT CastVT = CastVal.getValueType();
2759
2760 SDValue NewAtomic
2761 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2762 DAG.getVTList(CastVT, MVT::Other),
2763 { AM->getChain(), AM->getBasePtr(), CastVal },
2764 AM->getMemOperand());
2765
2766 SDValue Result = NewAtomic;
2767
2768 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2769 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2770 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2771 NewAtomic);
2772 }
2773
2774 // Legalize the chain result by replacing uses of the old value chain with the
2775 // new one
2776 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2777
2778 return Result;
2779
2780}
2781
2782//===----------------------------------------------------------------------===//
2783// Half Result Soft Promotion
2784//===----------------------------------------------------------------------===//
2785
2786void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2787 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2788 N->dump(&DAG));
2789 SDValue R = SDValue();
2790
2791 // See if the target wants to custom expand this node.
2792 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2793 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2794 return;
2795 }
2796
2797 switch (N->getOpcode()) {
2798 default:
2799#ifndef NDEBUG
2800 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2801 N->dump(&DAG); dbgs() << "\n";
2802#endif
2803 report_fatal_error("Do not know how to soft promote this operator's "
2804 "result!");
2805
2806 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2807 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2808 case ISD::EXTRACT_VECTOR_ELT:
2809 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2810 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2811 case ISD::STRICT_FP_ROUND:
2812 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2813
2814 // Unary FP Operations
2815 case ISD::FABS:
2816 case ISD::FCBRT:
2817 case ISD::FCEIL:
2818 case ISD::FCOS:
2819 case ISD::FEXP:
2820 case ISD::FEXP2:
2821 case ISD::FEXP10:
2822 case ISD::FFLOOR:
2823 case ISD::FLOG:
2824 case ISD::FLOG2:
2825 case ISD::FLOG10:
2826 case ISD::FNEARBYINT:
2827 case ISD::FNEG:
2828 case ISD::FREEZE:
2829 case ISD::FRINT:
2830 case ISD::FROUND:
2831 case ISD::FROUNDEVEN:
2832 case ISD::FSIN:
2833 case ISD::FSQRT:
2834 case ISD::FTRUNC:
2835 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2836
2837 // Binary FP Operations
2838 case ISD::FADD:
2839 case ISD::FDIV:
2840 case ISD::FMAXIMUM:
2841 case ISD::FMINIMUM:
2842 case ISD::FMAXNUM:
2843 case ISD::FMINNUM:
2844 case ISD::FMUL:
2845 case ISD::FPOW:
2846 case ISD::FREM:
2847 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2848
2849 case ISD::FMA: // FMA is same as FMAD
2850 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2851
2852 case ISD::FPOWI:
2853 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2854
2855 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2856
2857 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2858 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2859 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2860 case ISD::SINT_TO_FP:
2861 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2862 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2863 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2864 case ISD::VECREDUCE_FADD:
2865 case ISD::VECREDUCE_FMUL:
2866 case ISD::VECREDUCE_FMIN:
2867 case ISD::VECREDUCE_FMAX:
2868 case ISD::VECREDUCE_FMAXIMUM:
2869 case ISD::VECREDUCE_FMINIMUM:
2870 R = SoftPromoteHalfRes_VECREDUCE(N);
2871 break;
2872 case ISD::VECREDUCE_SEQ_FADD:
2873 case ISD::VECREDUCE_SEQ_FMUL:
2874 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2875 break;
2876 }
2877
2878 if (R.getNode())
2879 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2880}
2881
2882SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2883 return BitConvertToInteger(N->getOperand(0));
2884}
2885
2886SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2887 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2888
2889 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2890 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2891 MVT::i16);
2892}
2893
2894SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2895 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2896 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2897 NewOp.getValueType().getVectorElementType(), NewOp,
2898 N->getOperand(1));
2899}
2900
2901SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2902 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2903 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2904 SDLoc dl(N);
2905
2906 EVT LVT = LHS.getValueType();
2907 EVT RVT = RHS.getValueType();
2908
2909 unsigned LSize = LVT.getSizeInBits();
2910 unsigned RSize = RVT.getSizeInBits();
2911
2912 // First get the sign bit of second operand.
2913 SDValue SignBit = DAG.getNode(
2914 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2915 DAG.getConstant(RSize - 1, dl,
2916 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2917 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2918
2919 // Shift right or sign-extend it if the two operands have different types.
2920 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2921 if (SizeDiff > 0) {
2922 SignBit =
2923 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2924 DAG.getConstant(SizeDiff, dl,
2925 TLI.getShiftAmountTy(SignBit.getValueType(),
2926 DAG.getDataLayout())));
2927 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2928 } else if (SizeDiff < 0) {
2929 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2930 SignBit =
2931 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2932 DAG.getConstant(-SizeDiff, dl,
2933 TLI.getShiftAmountTy(SignBit.getValueType(),
2934 DAG.getDataLayout())));
2935 }
2936
2937 // Clear the sign bit of the first operand.
2938 SDValue Mask = DAG.getNode(
2939 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2940 DAG.getConstant(LSize - 1, dl,
2941 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2942 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2943 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2944
2945 // Or the value with the sign bit.
2946 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2947}
2948
2949SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2950 EVT OVT = N->getValueType(0);
2951 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2952 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2953 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2954 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2955 SDLoc dl(N);
2956
2957 // Promote to the larger FP type.
2958 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2959 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2960 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2961 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2962
2963 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2964
2965 // Convert back to FP16 as an integer.
2966 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2967}
2968
2969SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2970 EVT OVT = N->getValueType(0);
2971 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2972 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2973 SDValue Op1 = N->getOperand(1);
2974 SDLoc dl(N);
2975
2976 // Promote to the larger FP type.
2977 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2978
2979 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2980
2981 // Convert back to FP16 as an integer.
2982 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2983}
2984
2985SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2986 EVT OVT = N->getValueType(0);
2987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2988 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2989 SDLoc dl(N);
2990
2991 // Promote to the larger FP type.
2992 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2993
2994 SDValue Res = DAG.getNode(N->getOpcode(), dl,
2995 DAG.getVTList(NVT, N->getValueType(1)), Op);
2996
2997 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2998
2999 // Convert back to FP16 as an integer.
3000 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3001}
3002
3003SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3004 EVT RVT = N->getValueType(0);
3005 EVT SVT = N->getOperand(0).getValueType();
3006
3007 if (N->isStrictFPOpcode()) {
3008 // FIXME: assume we only have two f16 variants for now.
3009 unsigned Opcode;
3010 if (RVT == MVT::f16)
3011 Opcode = ISD::STRICT_FP_TO_FP16;
3012 else if (RVT == MVT::bf16)
3013 Opcode = ISD::STRICT_FP_TO_BF16;
3014 else
3015 llvm_unreachable("unknown half type");
3016 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3017 {N->getOperand(0), N->getOperand(1)});
3018 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3019 return Res;
3020 }
3021
3022 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3023 N->getOperand(0));
3024}
3025
3026SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3027 LoadSDNode *L = cast<LoadSDNode>(N);
3028
3029 // Load the value as an integer value with the same number of bits.
3030 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3031 SDValue NewL =
3032 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3033 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3034 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3035 L->getMemOperand()->getFlags(), L->getAAInfo());
3036 // Legalize the chain result by replacing uses of the old value chain with the
3037 // new one
3038 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3039 return NewL;
3040}
3041
3042SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3043 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3044 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3045 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3046 Op2);
3047}
3048
3049SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3050 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3051 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3052 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3053 N->getOperand(0), N->getOperand(1), Op2, Op3,
3054 N->getOperand(4));
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3058 EVT OVT = N->getValueType(0);
3059 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3060 SDLoc dl(N);
3061
3062 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3063
3064 // Round the value to the softened type.
3065 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3066}
3067
3068SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3069 return DAG.getUNDEF(MVT::i16);
3070}
3071
3072SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3073 EVT OVT = N->getValueType(0);
3074 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3075 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3076 SDLoc dl(N);
3077
3078 // Promote to the larger FP type.
3079 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3080
3081 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3082
3083 // Convert back to FP16 as an integer.
3084 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3085}
3086
3087SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3088 EVT OVT = N->getValueType(0);
3089 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3090 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3091 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3092 SDLoc dl(N);
3093
3094 // Promote to the larger FP type.
3095 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3096 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3097 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3098
3099 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3100
3101 // Convert back to FP16 as an integer.
3102 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3103}
3104
3105SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3106 // Expand and soften recursively.
3107 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3108 return SDValue();
3109}
3110
3111SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3112 // Expand and soften.
3113 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3114 return SDValue();
3115}
3116
3117//===----------------------------------------------------------------------===//
3118// Half Operand Soft Promotion
3119//===----------------------------------------------------------------------===//
3120
3121bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3122 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3123 N->dump(&DAG));
3124 SDValue Res = SDValue();
3125
3126 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3127 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3128 return false;
3129 }
3130
3131 // Nodes that use a promotion-requiring floating point operand, but doesn't
3132 // produce a soft promotion-requiring floating point result, need to be
3133 // legalized to use the soft promoted float operand. Nodes that produce at
3134 // least one soft promotion-requiring floating point result have their
3135 // operands legalized as a part of PromoteFloatResult.
3136 switch (N->getOpcode()) {
3137 default:
3138 #ifndef NDEBUG
3139 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3140 N->dump(&DAG); dbgs() << "\n";
3141 #endif
3142 report_fatal_error("Do not know how to soft promote this operator's "
3143 "operand!");
3144
3145 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3146 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3147 case ISD::FP_TO_SINT:
3148 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3149 case ISD::FP_TO_SINT_SAT:
3150 case ISD::FP_TO_UINT_SAT:
3151 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3152 case ISD::STRICT_FP_EXTEND:
3153 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3154 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3155 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3156 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3157 case ISD::STACKMAP:
3158 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3159 break;
3160 case ISD::PATCHPOINT:
3161 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3162 break;
3163 }
3164
3165 if (!Res.getNode())
3166 return false;
3167
3168 assert(Res.getNode() != N && "Expected a new node!");
3169
3170 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3171 "Invalid operand expansion");
3172
3173 ReplaceValueWith(SDValue(N, 0), Res);
3174 return false;
3175}
3176
3177SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3178 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3179
3180 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3181}
3182
3183SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3184 unsigned OpNo) {
3185 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3186 SDValue Op1 = N->getOperand(1);
3187 EVT RVT = Op1.getValueType();
3188 SDLoc dl(N);
3189
3190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3191
3192 Op1 = GetSoftPromotedHalf(Op1);
3193 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3194
3195 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3196 Op1);
3197}
3198
3199SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3200 EVT RVT = N->getValueType(0);
3201 bool IsStrict = N->isStrictFPOpcode();
3202 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3203 EVT SVT = Op.getValueType();
3204 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3205
3206 if (IsStrict) {
3207 unsigned Opcode;
3208 if (SVT == MVT::f16)
3209 Opcode = ISD::STRICT_FP16_TO_FP;
3210 else if (SVT == MVT::bf16)
3211 Opcode = ISD::STRICT_BF16_TO_FP;
3212 else
3213 llvm_unreachable("unknown half type");
3214 SDValue Res =
3215 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3216 {N->getOperand(0), Op});
3217 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3218 ReplaceValueWith(SDValue(N, 0), Res);
3219 return SDValue();
3220 }
3221
3222 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3223}
3224
3225SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3226 EVT RVT = N->getValueType(0);
3227 SDValue Op = N->getOperand(0);
3228 EVT SVT = Op.getValueType();
3229 SDLoc dl(N);
3230
3231 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3232
3233 Op = GetSoftPromotedHalf(Op);
3234
3235 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3236
3237 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3238}
3239
3240SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3241 EVT RVT = N->getValueType(0);
3242 SDValue Op = N->getOperand(0);
3243 EVT SVT = Op.getValueType();
3244 SDLoc dl(N);
3245
3246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3247
3248 Op = GetSoftPromotedHalf(Op);
3249
3250 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3251
3252 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3253 N->getOperand(1));
3254}
3255
3256SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3257 unsigned OpNo) {
3258 assert(OpNo == 0 && "Can only soften the comparison values");
3259 SDValue Op0 = N->getOperand(0);
3260 SDValue Op1 = N->getOperand(1);
3261 SDLoc dl(N);
3262
3263 EVT SVT = Op0.getValueType();
3264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3265
3266 Op0 = GetSoftPromotedHalf(Op0);
3267 Op1 = GetSoftPromotedHalf(Op1);
3268
3269 // Promote to the larger FP type.
3270 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3271 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3272 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3273
3274 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3275 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3276}
3277
3278SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3279 SDValue Op0 = N->getOperand(0);
3280 SDValue Op1 = N->getOperand(1);
3281 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3282 SDLoc dl(N);
3283
3284 EVT SVT = Op0.getValueType();
3285 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3286
3287 Op0 = GetSoftPromotedHalf(Op0);
3288 Op1 = GetSoftPromotedHalf(Op1);
3289
3290 // Promote to the larger FP type.
3291 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3292 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3293 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3294
3295 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3296}
3297
3298SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3299 assert(OpNo == 1 && "Can only soften the stored value!");
3300 StoreSDNode *ST = cast<StoreSDNode>(N);
3301 SDValue Val = ST->getValue();
3302 SDLoc dl(N);
3303
3304 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3305 SDValue Promoted = GetSoftPromotedHalf(Val);
3306 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3307 ST->getMemOperand());
3308}
3309
3310SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3311 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3312 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3313 SDValue Op = N->getOperand(OpNo);
3314 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3315 SDValue NewNode =
3316 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3317
3318 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3319 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3320
3321 return SDValue(); // Signal that we replaced the node ourselves.
3322}
3323
3324SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3325 unsigned OpNo) {
3326 assert(OpNo >= 7);
3327 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3328 SDValue Op = N->getOperand(OpNo);
3329 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3330 SDValue NewNode =
3331 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3332
3333 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3334 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3335
3336 return SDValue(); // Signal that we replaced the node ourselves.
3337}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:74
Expansion
DXIL Intrinsic Expansion
Definition: DXILIntrinsicExpansion.cpp:323
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:354
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:1039
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2183
GetPromotionOpcodeStrict
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2197
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:76
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1470
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1492
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1690
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:239
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:279
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2396
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:58
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:144
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:146
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1382
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1129
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:477
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:1006
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:1009
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1164
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8699
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1236
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10085
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1207
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1789
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8682
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8403
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1098
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2341
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:471
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1604
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9702
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8783
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8732
llvm::SelectionDAG::EVTToAPFloatSemantics
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Definition: SelectionDAG.h:1798
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1246
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1729
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9708
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:478
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:468
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:1990
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:484
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
Definition: SelectionDAG.cpp:1734
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2459
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10175
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:553
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2424
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:566
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:1017
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1136
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3423
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:290
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:145
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:10775
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:10826
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:10735
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:8308
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1345
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1037
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1361
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
llvm::ISD::VECREDUCE_FMAXIMUM
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1365
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1358
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1362
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1083
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
llvm::ISD::STRICT_FPOWI
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
llvm::ISD::EXTRACT_ELEMENT
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1231
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
llvm::ISD::STRICT_BF16_TO_FP
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:924
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:990
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
llvm::ISD::STRICT_FP_TO_BF16
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1359
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1268
llvm::ISD::FFREXP
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
llvm::ISD::STRICT_FLDEXP
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
llvm::ISD::STRICT_LLROUND
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
llvm::ISD::VECREDUCE_FMINIMUM
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1366
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1152
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1346
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3152
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3145
llvm::ISD::isUNINDEXEDStore
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
Definition: SelectionDAGNodes.h:3159
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1529
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3114
llvm::RTLIB::getPOWI
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:551
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:461
llvm::RTLIB::getLDEXP
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:556
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:507
llvm::RTLIB::getFREXP
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:561
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:412
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:363
llvm::RTLIB::getFPEXT
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:285
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:320
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::tgtok::FalseVal
@ FalseVal
Definition: TGLexer.h:59
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Offset
@ Offset
Definition: DWP.cpp:456
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
llvm::Op
DWARFExpression::Operation Op
Definition: DWARFExpression.cpp:22
raw_ostream.h
N
#define N
llvm::APFloatBase::PPCDoubleDouble
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::EVT::bitsGE
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1062
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::TargetLowering::MakeLibCallOptions
This structure is used to pass arguments to makeLibCall function.
Definition: TargetLowering.h:4657
llvm::TargetLowering::MakeLibCallOptions::setSExt
MakeLibCallOptions & setSExt(bool Value=true)
Definition: TargetLowering.h:4672
llvm::TargetLowering::MakeLibCallOptions::setTypeListBeforeSoften
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
Definition: TargetLowering.h:4692
Generated on Tue Apr 23 2024 05:56:57 for LLVM by doxygen 1.9.6