LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::STRICT_FCEIL:
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::STRICT_FCOS:
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::STRICT_FDIV:
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::STRICT_FEXP:
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::STRICT_FEXP2:
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::STRICT_FFLOOR:
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::STRICT_FLOG:
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::STRICT_FLOG2:
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::STRICT_FLOG10:
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::STRICT_FNEARBYINT:
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::STRICT_FP_EXTEND:
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::STRICT_FP_ROUND:
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::STRICT_FPOWI:
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
122 case ISD::STRICT_FRINT:
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
124 case ISD::STRICT_FROUND:
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
126 case ISD::STRICT_FROUNDEVEN:
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
130 case ISD::STRICT_FSQRT:
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
134 case ISD::STRICT_FTRUNC:
135 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
136 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
137 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
142 case ISD::STRICT_SINT_TO_FP:
143 case ISD::STRICT_UINT_TO_FP:
144 case ISD::SINT_TO_FP:
145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
148 case ISD::VECREDUCE_FADD:
149 case ISD::VECREDUCE_FMUL:
150 case ISD::VECREDUCE_FMIN:
151 case ISD::VECREDUCE_FMAX:
152 case ISD::VECREDUCE_FMAXIMUM:
153 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
154 case ISD::VECREDUCE_SEQ_FADD:
155 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
156 // clang-format on
157 }
158
159 // If R is null, the sub-method took care of registering the result.
160 if (R.getNode()) {
161 assert(R.getNode() != N);
162 SetSoftenedFloat(SDValue(N, ResNo), R);
163 }
164}
165
166SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
167 bool IsStrict = N->isStrictFPOpcode();
168 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
169 unsigned Offset = IsStrict ? 1 : 0;
170 assert(N->getNumOperands() == (1 + Offset) &&
171 "Unexpected number of operands!");
172 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
173 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
174 TargetLowering::MakeLibCallOptions CallOptions;
175 EVT OpVT = N->getOperand(0 + Offset).getValueType();
176 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
177 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
178 CallOptions, SDLoc(N),
179 Chain);
180 if (IsStrict)
181 ReplaceValueWith(SDValue(N, 1), Tmp.second);
182 return Tmp.first;
183}
184
185SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
186 bool IsStrict = N->isStrictFPOpcode();
187 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
188 unsigned Offset = IsStrict ? 1 : 0;
189 assert(N->getNumOperands() == (2 + Offset) &&
190 "Unexpected number of operands!");
191 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
192 GetSoftenedFloat(N->getOperand(1 + Offset)) };
193 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
194 TargetLowering::MakeLibCallOptions CallOptions;
195 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
196 N->getOperand(1 + Offset).getValueType() };
197 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
198 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
199 CallOptions, SDLoc(N),
200 Chain);
201 if (IsStrict)
202 ReplaceValueWith(SDValue(N, 1), Tmp.second);
203 return Tmp.first;
204}
205
206SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
207 return BitConvertToInteger(N->getOperand(0));
208}
209
210SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
211 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
212 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
213 GetSoftenedFloat(N->getOperand(0)));
214}
215
216SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
217 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
218 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
219 GetSoftenedFloat(N->getOperand(0)));
220 return NewFence;
221}
222
223SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
224 unsigned ResNo) {
225 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
226 return BitConvertToInteger(Op);
227}
228
229SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
230 // Convert the inputs to integers, and build a new pair out of them.
231 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
232 TLI.getTypeToTransformTo(*DAG.getContext(),
233 N->getValueType(0)),
234 BitConvertToInteger(N->getOperand(0)),
235 BitConvertToInteger(N->getOperand(1)));
236}
237
238SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
239 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
240 // In ppcf128, the high 64 bits are always first in memory regardless
241 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
242 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
243 // way. However, APInt's are serialized in an Endian-sensitive fashion,
244 // so on big-Endian targets, the two doubles are output in the wrong
245 // order. Fix this by manually flipping the order of the high 64 bits
246 // and the low 64 bits here.
247 if (DAG.getDataLayout().isBigEndian() &&
248 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
249 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
250 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
251 APInt Val(128, words);
252 return DAG.getConstant(Val, SDLoc(CN),
253 TLI.getTypeToTransformTo(*DAG.getContext(),
254 CN->getValueType(0)));
255 } else {
256 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
257 TLI.getTypeToTransformTo(*DAG.getContext(),
258 CN->getValueType(0)));
259 }
260}
261
262SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
263 SDValue Src = N->getOperand(0);
264 assert(Src.getValueType() == MVT::ppcf128 &&
265 "In floats only ppcf128 can be extracted by element!");
266 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
267 N->getValueType(0).changeTypeToInteger(),
268 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
269}
270
271SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
272 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
273 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
274 NewOp.getValueType().getVectorElementType(),
275 NewOp, N->getOperand(1));
276}
277
278SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
279 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
280 unsigned Size = NVT.getSizeInBits();
281
282 // Mask = ~(1 << (Size-1))
283 APInt API = APInt::getAllOnes(Size);
284 API.clearBit(Size - 1);
285 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
286 SDValue Op = GetSoftenedFloat(N->getOperand(0));
287 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
288}
289
290SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
291 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
292 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
293 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
294 RTLIB::FMIN_F32,
295 RTLIB::FMIN_F64,
296 RTLIB::FMIN_F80,
297 RTLIB::FMIN_F128,
298 RTLIB::FMIN_PPCF128));
299}
300
301SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
302 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
303 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
304 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
305 RTLIB::FMAX_F32,
306 RTLIB::FMAX_F64,
307 RTLIB::FMAX_F80,
308 RTLIB::FMAX_F128,
309 RTLIB::FMAX_PPCF128));
310}
311
312SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
313 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
314 RTLIB::ADD_F32,
315 RTLIB::ADD_F64,
316 RTLIB::ADD_F80,
317 RTLIB::ADD_F128,
318 RTLIB::ADD_PPCF128));
319}
320
321SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
322 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
323 RTLIB::CBRT_F32,
324 RTLIB::CBRT_F64,
325 RTLIB::CBRT_F80,
326 RTLIB::CBRT_F128,
327 RTLIB::CBRT_PPCF128));
328}
329
330SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
331 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
332 RTLIB::CEIL_F32,
333 RTLIB::CEIL_F64,
334 RTLIB::CEIL_F80,
335 RTLIB::CEIL_F128,
336 RTLIB::CEIL_PPCF128));
337}
338
339SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
340 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
341 SDValue RHS = BitConvertToInteger(N->getOperand(1));
342 SDLoc dl(N);
343
344 EVT LVT = LHS.getValueType();
345 EVT RVT = RHS.getValueType();
346
347 unsigned LSize = LVT.getSizeInBits();
348 unsigned RSize = RVT.getSizeInBits();
349
350 // First get the sign bit of second operand.
351 SDValue SignBit = DAG.getNode(
352 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
353 DAG.getConstant(RSize - 1, dl,
354 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
355 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
356
357 // Shift right or sign-extend it if the two operands have different types.
358 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
359 if (SizeDiff > 0) {
360 SignBit =
361 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
362 DAG.getConstant(SizeDiff, dl,
363 TLI.getShiftAmountTy(SignBit.getValueType(),
364 DAG.getDataLayout())));
365 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
366 } else if (SizeDiff < 0) {
367 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
368 SignBit =
369 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
370 DAG.getConstant(-SizeDiff, dl,
371 TLI.getShiftAmountTy(SignBit.getValueType(),
372 DAG.getDataLayout())));
373 }
374
375 // Clear the sign bit of the first operand.
376 SDValue Mask = DAG.getNode(
377 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
378 DAG.getConstant(LSize - 1, dl,
379 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
380 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
381 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
382
383 // Or the value with the sign bit.
384 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
385}
386
387SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
388 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
389 RTLIB::COS_F32,
390 RTLIB::COS_F64,
391 RTLIB::COS_F80,
392 RTLIB::COS_F128,
393 RTLIB::COS_PPCF128));
394}
395
396SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
397 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
398 RTLIB::DIV_F32,
399 RTLIB::DIV_F64,
400 RTLIB::DIV_F80,
401 RTLIB::DIV_F128,
402 RTLIB::DIV_PPCF128));
403}
404
405SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
406 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
407 RTLIB::EXP_F32,
408 RTLIB::EXP_F64,
409 RTLIB::EXP_F80,
410 RTLIB::EXP_F128,
411 RTLIB::EXP_PPCF128));
412}
413
414SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
415 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
416 RTLIB::EXP2_F32,
417 RTLIB::EXP2_F64,
418 RTLIB::EXP2_F80,
419 RTLIB::EXP2_F128,
420 RTLIB::EXP2_PPCF128));
421}
422
423SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
424 return SoftenFloatRes_Unary(
425 N,
426 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
427 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
428}
429
430SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
431 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
432 RTLIB::FLOOR_F32,
433 RTLIB::FLOOR_F64,
434 RTLIB::FLOOR_F80,
435 RTLIB::FLOOR_F128,
436 RTLIB::FLOOR_PPCF128));
437}
438
439SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
440 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
441 RTLIB::LOG_F32,
442 RTLIB::LOG_F64,
443 RTLIB::LOG_F80,
444 RTLIB::LOG_F128,
445 RTLIB::LOG_PPCF128));
446}
447
448SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
449 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
450 RTLIB::LOG2_F32,
451 RTLIB::LOG2_F64,
452 RTLIB::LOG2_F80,
453 RTLIB::LOG2_F128,
454 RTLIB::LOG2_PPCF128));
455}
456
457SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
458 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
459 RTLIB::LOG10_F32,
460 RTLIB::LOG10_F64,
461 RTLIB::LOG10_F80,
462 RTLIB::LOG10_F128,
463 RTLIB::LOG10_PPCF128));
464}
465
466SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
467 bool IsStrict = N->isStrictFPOpcode();
468 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
469 unsigned Offset = IsStrict ? 1 : 0;
470 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
471 GetSoftenedFloat(N->getOperand(1 + Offset)),
472 GetSoftenedFloat(N->getOperand(2 + Offset)) };
473 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
474 TargetLowering::MakeLibCallOptions CallOptions;
475 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
476 N->getOperand(1 + Offset).getValueType(),
477 N->getOperand(2 + Offset).getValueType() };
478 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
479 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
480 GetFPLibCall(N->getValueType(0),
481 RTLIB::FMA_F32,
482 RTLIB::FMA_F64,
483 RTLIB::FMA_F80,
484 RTLIB::FMA_F128,
485 RTLIB::FMA_PPCF128),
486 NVT, Ops, CallOptions, SDLoc(N), Chain);
487 if (IsStrict)
488 ReplaceValueWith(SDValue(N, 1), Tmp.second);
489 return Tmp.first;
490}
491
492SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
493 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
494 RTLIB::MUL_F32,
495 RTLIB::MUL_F64,
496 RTLIB::MUL_F80,
497 RTLIB::MUL_F128,
498 RTLIB::MUL_PPCF128));
499}
500
501SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
502 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
503 RTLIB::NEARBYINT_F32,
504 RTLIB::NEARBYINT_F64,
505 RTLIB::NEARBYINT_F80,
506 RTLIB::NEARBYINT_F128,
507 RTLIB::NEARBYINT_PPCF128));
508}
509
510SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
511 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
512 SDLoc dl(N);
513
514 // Expand Y = FNEG(X) -> Y = X ^ sign mask
515 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
516 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
517 DAG.getConstant(SignMask, dl, NVT));
518}
519
520SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
521 bool IsStrict = N->isStrictFPOpcode();
522 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
523 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
524
525 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
526
527 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
528 Op = GetPromotedFloat(Op);
529 // If the promotion did the FP_EXTEND to the destination type for us,
530 // there's nothing left to do here.
531 if (Op.getValueType() == N->getValueType(0)) {
532 if (IsStrict)
533 ReplaceValueWith(SDValue(N, 1), Chain);
534 return BitConvertToInteger(Op);
535 }
536 }
537
538 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
539 // -> f32, so proceed in two stages. Also, it's entirely possible for both
540 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
541 // than FP16_TO_FP.
542 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
543 N->getValueType(0) != MVT::f32) {
544 if (IsStrict) {
545 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
546 { MVT::f32, MVT::Other }, { Chain, Op });
547 Chain = Op.getValue(1);
548 } else {
549 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
550 }
551 }
552
553 if (Op.getValueType() == MVT::bf16) {
554 // FIXME: Need ReplaceValueWith on chain in strict case
555 return SoftenFloatRes_BF16_TO_FP(N);
556 }
557
558 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
559 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
560 TargetLowering::MakeLibCallOptions CallOptions;
561 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
562 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
563 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
564 CallOptions, SDLoc(N),
565 Chain);
566 if (IsStrict)
567 ReplaceValueWith(SDValue(N, 1), Tmp.second);
568 return Tmp.first;
569}
570
571// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
572// nodes?
573SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
574 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
575 SDValue Op = N->getOperand(0);
576 TargetLowering::MakeLibCallOptions CallOptions;
577 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
578 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
579 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
580 CallOptions, SDLoc(N)).first;
581 if (N->getValueType(0) == MVT::f32)
582 return Res32;
583
584 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
585 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
586 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
587 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
588}
589
590// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
591// nodes?
592SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
593 assert(N->getValueType(0) == MVT::f32 &&
594 "Can only soften BF16_TO_FP with f32 result");
595 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
596 SDValue Op = N->getOperand(0);
597 SDLoc DL(N);
598 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
599 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
600 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
601 DAG.getShiftAmountConstant(16, NVT, DL));
602 return Res;
603}
604
605SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
606 bool IsStrict = N->isStrictFPOpcode();
607 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
608 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
609 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
610 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
611 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
612 TargetLowering::MakeLibCallOptions CallOptions;
613 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
614 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
615 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
616 CallOptions, SDLoc(N),
617 Chain);
618 if (IsStrict)
619 ReplaceValueWith(SDValue(N, 1), Tmp.second);
620 return Tmp.first;
621}
622
623SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
624 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
625 RTLIB::POW_F32,
626 RTLIB::POW_F64,
627 RTLIB::POW_F80,
628 RTLIB::POW_F128,
629 RTLIB::POW_PPCF128));
630}
631
632SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
633 bool IsStrict = N->isStrictFPOpcode();
634 unsigned Offset = IsStrict ? 1 : 0;
635 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
636 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
637 "Unsupported power type!");
638 bool IsPowI =
639 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
640
641 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
642 : RTLIB::getLDEXP(N->getValueType(0));
643 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
644 if (!TLI.getLibcallName(LC)) {
645 // Some targets don't have a powi libcall; use pow instead.
646 // FIXME: Implement this if some target needs it.
647 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
648 return DAG.getUNDEF(N->getValueType(0));
649 }
650
651 if (DAG.getLibInfo().getIntSize() !=
652 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
653 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
654 // would use the wrong type for the argument.
655 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
656 return DAG.getUNDEF(N->getValueType(0));
657 }
658
659 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
660 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
661 N->getOperand(1 + Offset) };
662 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
663 TargetLowering::MakeLibCallOptions CallOptions;
664 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
665 N->getOperand(1 + Offset).getValueType() };
666 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
667 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
668 CallOptions, SDLoc(N),
669 Chain);
670 if (IsStrict)
671 ReplaceValueWith(SDValue(N, 1), Tmp.second);
672 return Tmp.first;
673}
674
675SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
676 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
677 EVT VT0 = N->getValueType(0);
678 EVT VT1 = N->getValueType(1);
679 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
680
681 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
682 // If the exponent does not match with sizeof(int) a libcall would use the
683 // wrong type for the argument.
684 // TODO: Should be able to handle mismatches.
685 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
686 return DAG.getUNDEF(N->getValueType(0));
687 }
688
689 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
690 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
691
692 SDLoc DL(N);
693
694 TargetLowering::MakeLibCallOptions CallOptions;
695 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
696 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
697
698 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
699 // but we only really need to handle the 0th one for softening anyway.
700 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
701
702 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
703 /*Chain=*/SDValue());
704 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
705 auto PtrInfo =
706 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
707
708 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
709
710 ReplaceValueWith(SDValue(N, 1), LoadExp);
711 return ReturnVal;
712}
713
714SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
715 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
716 RTLIB::REM_F32,
717 RTLIB::REM_F64,
718 RTLIB::REM_F80,
719 RTLIB::REM_F128,
720 RTLIB::REM_PPCF128));
721}
722
723SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
724 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
725 RTLIB::RINT_F32,
726 RTLIB::RINT_F64,
727 RTLIB::RINT_F80,
728 RTLIB::RINT_F128,
729 RTLIB::RINT_PPCF128));
730}
731
732SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
733 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
734 RTLIB::ROUND_F32,
735 RTLIB::ROUND_F64,
736 RTLIB::ROUND_F80,
737 RTLIB::ROUND_F128,
738 RTLIB::ROUND_PPCF128));
739}
740
741SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
742 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
743 RTLIB::ROUNDEVEN_F32,
744 RTLIB::ROUNDEVEN_F64,
745 RTLIB::ROUNDEVEN_F80,
746 RTLIB::ROUNDEVEN_F128,
747 RTLIB::ROUNDEVEN_PPCF128));
748}
749
750SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
751 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
752 RTLIB::SIN_F32,
753 RTLIB::SIN_F64,
754 RTLIB::SIN_F80,
755 RTLIB::SIN_F128,
756 RTLIB::SIN_PPCF128));
757}
758
759SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
760 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
761 RTLIB::SQRT_F32,
762 RTLIB::SQRT_F64,
763 RTLIB::SQRT_F80,
764 RTLIB::SQRT_F128,
765 RTLIB::SQRT_PPCF128));
766}
767
768SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
769 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
770 RTLIB::SUB_F32,
771 RTLIB::SUB_F64,
772 RTLIB::SUB_F80,
773 RTLIB::SUB_F128,
774 RTLIB::SUB_PPCF128));
775}
776
777SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
778 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
779 RTLIB::TRUNC_F32,
780 RTLIB::TRUNC_F64,
781 RTLIB::TRUNC_F80,
782 RTLIB::TRUNC_F128,
783 RTLIB::TRUNC_PPCF128));
784}
785
786SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
787 LoadSDNode *L = cast<LoadSDNode>(N);
788 EVT VT = N->getValueType(0);
789 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
790 SDLoc dl(N);
791
792 auto MMOFlags =
793 L->getMemOperand()->getFlags() &
794 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
795 SDValue NewL;
796 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
797 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
798 L->getChain(), L->getBasePtr(), L->getOffset(),
799 L->getPointerInfo(), NVT, L->getOriginalAlign(),
800 MMOFlags, L->getAAInfo());
801 // Legalized the chain result - switch anything that used the old chain to
802 // use the new one.
803 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
804 return NewL;
805 }
806
807 // Do a non-extending load followed by FP_EXTEND.
808 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
809 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
810 L->getPointerInfo(), L->getMemoryVT(),
811 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
812 // Legalized the chain result - switch anything that used the old chain to
813 // use the new one.
814 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
815 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
816 return BitConvertToInteger(ExtendNode);
817}
818
819SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
820 AtomicSDNode *L = cast<AtomicSDNode>(N);
821 EVT VT = N->getValueType(0);
822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
823 SDLoc dl(N);
824
825 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
826 SDValue NewL =
827 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
828 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
829
830 // Legalized the chain result - switch anything that used the old chain to
831 // use the new one.
832 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
833 return NewL;
834 }
835
836 report_fatal_error("softening fp extending atomic load not handled");
837}
838
839SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
840 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
841 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
842 return DAG.getSelect(SDLoc(N),
843 LHS.getValueType(), N->getOperand(0), LHS, RHS);
844}
845
846SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
847 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
848 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
849 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
850 LHS.getValueType(), N->getOperand(0),
851 N->getOperand(1), LHS, RHS, N->getOperand(4));
852}
853
854SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
855 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
856 N->getValueType(0)));
857}
858
859SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
860 SDValue Chain = N->getOperand(0); // Get the chain.
861 SDValue Ptr = N->getOperand(1); // Get the pointer.
862 EVT VT = N->getValueType(0);
863 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
864 SDLoc dl(N);
865
866 SDValue NewVAARG;
867 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
868 N->getConstantOperandVal(3));
869
870 // Legalized the chain result - switch anything that used the old chain to
871 // use the new one.
872 if (N != NewVAARG.getValue(1).getNode())
873 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
874 return NewVAARG;
875}
876
877SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
878 bool IsStrict = N->isStrictFPOpcode();
879 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
880 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
881 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
882 EVT RVT = N->getValueType(0);
883 EVT NVT = EVT();
884 SDLoc dl(N);
885
886 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
887 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
888 // match. Look for an appropriate libcall.
889 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
890 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
891 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
892 NVT = (MVT::SimpleValueType)t;
893 // The source needs to big enough to hold the operand.
894 if (NVT.bitsGE(SVT))
895 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
896 }
897 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
898
899 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
900 // Sign/zero extend the argument if the libcall takes a larger type.
901 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
902 NVT, N->getOperand(IsStrict ? 1 : 0));
903 TargetLowering::MakeLibCallOptions CallOptions;
904 CallOptions.setSExt(Signed);
905 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
906 std::pair<SDValue, SDValue> Tmp =
907 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
908 Op, CallOptions, dl, Chain);
909
910 if (IsStrict)
911 ReplaceValueWith(SDValue(N, 1), Tmp.second);
912 return Tmp.first;
913}
914
915SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
916 // Expand and soften recursively.
917 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
918 return SDValue();
919}
920
921SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
922 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
923 return SDValue();
924}
925
926//===----------------------------------------------------------------------===//
927// Convert Float Operand to Integer
928//===----------------------------------------------------------------------===//
929
930bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
931 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
932 SDValue Res = SDValue();
933
934 switch (N->getOpcode()) {
935 default:
936#ifndef NDEBUG
937 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
938 N->dump(&DAG); dbgs() << "\n";
939#endif
940 report_fatal_error("Do not know how to soften this operator's operand!");
941
942 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
943 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
944 case ISD::STRICT_FP_TO_FP16:
945 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
946 case ISD::FP_TO_BF16:
947 case ISD::STRICT_FP_TO_BF16:
948 case ISD::STRICT_FP_ROUND:
949 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
950 case ISD::STRICT_FP_TO_SINT:
951 case ISD::STRICT_FP_TO_UINT:
952 case ISD::FP_TO_SINT:
953 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
954 case ISD::FP_TO_SINT_SAT:
955 case ISD::FP_TO_UINT_SAT:
956 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
957 case ISD::STRICT_LROUND:
958 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
959 case ISD::STRICT_LLROUND:
960 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
961 case ISD::STRICT_LRINT:
962 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
963 case ISD::STRICT_LLRINT:
964 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
965 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
966 case ISD::STRICT_FSETCC:
967 case ISD::STRICT_FSETCCS:
968 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
969 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
970 case ISD::ATOMIC_STORE:
971 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
972 break;
973 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
974 }
975
976 // If the result is null, the sub-method took care of registering results etc.
977 if (!Res.getNode()) return false;
978
979 // If the result is N, the sub-method updated N in place. Tell the legalizer
980 // core about this to re-analyze.
981 if (Res.getNode() == N)
982 return true;
983
984 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
985 "Invalid operand softening");
986
987 ReplaceValueWith(SDValue(N, 0), Res);
988 return false;
989}
990
991SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
992 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
993
994 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
995}
996
997SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
998 // We actually deal with the partially-softened FP_TO_FP16 node too, which
999 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1000 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1001 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1002 N->getOpcode() == ISD::FP_TO_BF16 ||
1003 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1004 N->getOpcode() == ISD::STRICT_FP_ROUND);
1005
1006 bool IsStrict = N->isStrictFPOpcode();
1007 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1008 EVT SVT = Op.getValueType();
1009 EVT RVT = N->getValueType(0);
1010 EVT FloatRVT = RVT;
1011 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1012 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1013 FloatRVT = MVT::f16;
1014 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1015 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1016 FloatRVT = MVT::bf16;
1017
1018 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1019 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1020
1021 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1022 Op = GetSoftenedFloat(Op);
1023 TargetLowering::MakeLibCallOptions CallOptions;
1024 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1025 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1026 CallOptions, SDLoc(N),
1027 Chain);
1028 if (IsStrict) {
1029 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1030 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1031 return SDValue();
1032 }
1033 return Tmp.first;
1034}
1035
1036SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1037 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1038 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1039
1040 EVT VT = NewLHS.getValueType();
1041 NewLHS = GetSoftenedFloat(NewLHS);
1042 NewRHS = GetSoftenedFloat(NewRHS);
1043 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1044 N->getOperand(2), N->getOperand(3));
1045
1046 // If softenSetCCOperands returned a scalar, we need to compare the result
1047 // against zero to select between true and false values.
1048 if (!NewRHS.getNode()) {
1049 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1050 CCCode = ISD::SETNE;
1051 }
1052
1053 // Update N to have the operands specified.
1054 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1055 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1056 N->getOperand(4)),
1057 0);
1058}
1059
1060// Even if the result type is legal, no libcall may exactly match. (e.g. We
1061// don't have FP-i8 conversions) This helper method looks for an appropriate
1062// promoted libcall.
1063static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1064 bool Signed) {
1065 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1066 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1067 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1068 ++IntVT) {
1069 Promoted = (MVT::SimpleValueType)IntVT;
1070 // The type needs to big enough to hold the result.
1071 if (Promoted.bitsGE(RetVT))
1072 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1073 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1074 }
1075 return LC;
1076}
1077
1078SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1079 bool IsStrict = N->isStrictFPOpcode();
1080 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1081 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1082
1083 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1084 EVT SVT = Op.getValueType();
1085 EVT RVT = N->getValueType(0);
1086 EVT NVT = EVT();
1087 SDLoc dl(N);
1088
1089 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1090 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1091 // match, eg. we don't have fp -> i8 conversions.
1092 // Look for an appropriate libcall.
1093 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1094 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1095 "Unsupported FP_TO_XINT!");
1096
1097 Op = GetSoftenedFloat(Op);
1098 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1099 TargetLowering::MakeLibCallOptions CallOptions;
1100 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1101 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1102 CallOptions, dl, Chain);
1103
1104 // Truncate the result if the libcall returns a larger type.
1105 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1106
1107 if (!IsStrict)
1108 return Res;
1109
1110 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1111 ReplaceValueWith(SDValue(N, 0), Res);
1112 return SDValue();
1113}
1114
1115SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1116 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1117 return Res;
1118}
1119
1120SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1121 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1122 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1123
1124 EVT VT = NewLHS.getValueType();
1125 NewLHS = GetSoftenedFloat(NewLHS);
1126 NewRHS = GetSoftenedFloat(NewRHS);
1127 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1128 N->getOperand(0), N->getOperand(1));
1129
1130 // If softenSetCCOperands returned a scalar, we need to compare the result
1131 // against zero to select between true and false values.
1132 if (!NewRHS.getNode()) {
1133 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1134 CCCode = ISD::SETNE;
1135 }
1136
1137 // Update N to have the operands specified.
1138 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1139 N->getOperand(2), N->getOperand(3),
1140 DAG.getCondCode(CCCode)),
1141 0);
1142}
1143
1144SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1145 bool IsStrict = N->isStrictFPOpcode();
1146 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1147 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1148 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1149 ISD::CondCode CCCode =
1150 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1151
1152 EVT VT = Op0.getValueType();
1153 SDValue NewLHS = GetSoftenedFloat(Op0);
1154 SDValue NewRHS = GetSoftenedFloat(Op1);
1155 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1156 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1157
1158 // Update N to have the operands specified.
1159 if (NewRHS.getNode()) {
1160 if (IsStrict)
1161 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1162 NewRHS, DAG.getCondCode(CCCode));
1163 else
1164 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1165 DAG.getCondCode(CCCode)), 0);
1166 }
1167
1168 // Otherwise, softenSetCCOperands returned a scalar, use it.
1169 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1170 "Unexpected setcc expansion!");
1171
1172 if (IsStrict) {
1173 ReplaceValueWith(SDValue(N, 0), NewLHS);
1174 ReplaceValueWith(SDValue(N, 1), Chain);
1175 return SDValue();
1176 }
1177 return NewLHS;
1178}
1179
1180SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1181 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1182 assert(OpNo == 1 && "Can only soften the stored value!");
1183 StoreSDNode *ST = cast<StoreSDNode>(N);
1184 SDValue Val = ST->getValue();
1185 SDLoc dl(N);
1186
1187 if (ST->isTruncatingStore())
1188 // Do an FP_ROUND followed by a non-truncating store.
1189 Val = BitConvertToInteger(
1190 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1191 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1192 else
1193 Val = GetSoftenedFloat(Val);
1194
1195 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1196 ST->getMemOperand());
1197}
1198
1199SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1200 assert(OpNo == 1 && "Can only soften the stored value!");
1201 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1202 SDValue Val = ST->getVal();
1203 EVT VT = Val.getValueType();
1204 SDLoc dl(N);
1205
1206 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1207
1208 SDValue NewVal = GetSoftenedFloat(Val);
1209 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1210 ST->getBasePtr(), ST->getMemOperand());
1211}
1212
1213SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1214 SDValue LHS = N->getOperand(0);
1215 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1216 SDLoc dl(N);
1217
1218 EVT LVT = LHS.getValueType();
1219 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1220 EVT RVT = RHS.getValueType();
1221
1222 unsigned LSize = LVT.getSizeInBits();
1223 unsigned RSize = RVT.getSizeInBits();
1224
1225 // Shift right or sign-extend it if the two operands have different types.
1226 int SizeDiff = RSize - LSize;
1227 if (SizeDiff > 0) {
1228 RHS =
1229 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1230 DAG.getConstant(SizeDiff, dl,
1231 TLI.getShiftAmountTy(RHS.getValueType(),
1232 DAG.getDataLayout())));
1233 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1234 } else if (SizeDiff < 0) {
1235 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1236 RHS =
1237 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1238 DAG.getConstant(-SizeDiff, dl,
1239 TLI.getShiftAmountTy(RHS.getValueType(),
1240 DAG.getDataLayout())));
1241 }
1242
1243 RHS = DAG.getBitcast(LVT, RHS);
1244 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1245}
1246
1247SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1248 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1249 bool IsStrict = N->isStrictFPOpcode();
1250 unsigned Offset = IsStrict ? 1 : 0;
1251 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1252 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1253 TargetLowering::MakeLibCallOptions CallOptions;
1254 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1255 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1256 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1257 CallOptions, SDLoc(N),
1258 Chain);
1259 if (IsStrict) {
1260 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1261 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1262 return SDValue();
1263 }
1264
1265 return Tmp.first;
1266}
1267
1268SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1269 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1270 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1271 RTLIB::LROUND_F32,
1272 RTLIB::LROUND_F64,
1273 RTLIB::LROUND_F80,
1274 RTLIB::LROUND_F128,
1275 RTLIB::LROUND_PPCF128));
1276}
1277
1278SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1279 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1280 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1281 RTLIB::LLROUND_F32,
1282 RTLIB::LLROUND_F64,
1283 RTLIB::LLROUND_F80,
1284 RTLIB::LLROUND_F128,
1285 RTLIB::LLROUND_PPCF128));
1286}
1287
1288SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1289 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1290 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1291 RTLIB::LRINT_F32,
1292 RTLIB::LRINT_F64,
1293 RTLIB::LRINT_F80,
1294 RTLIB::LRINT_F128,
1295 RTLIB::LRINT_PPCF128));
1296}
1297
1298SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1299 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1300 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1301 RTLIB::LLRINT_F32,
1302 RTLIB::LLRINT_F64,
1303 RTLIB::LLRINT_F80,
1304 RTLIB::LLRINT_F128,
1305 RTLIB::LLRINT_PPCF128));
1306}
1307
1308//===----------------------------------------------------------------------===//
1309// Float Result Expansion
1310//===----------------------------------------------------------------------===//
1311
1312/// ExpandFloatResult - This method is called when the specified result of the
1313/// specified node is found to need expansion. At this point, the node may also
1314/// have invalid operands or may have other results that need promotion, we just
1315/// know that (at least) one result needs expansion.
1316void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1317 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1318 SDValue Lo, Hi;
1319 Lo = Hi = SDValue();
1320
1321 // See if the target wants to custom expand this node.
1322 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1323 return;
1324
1325 switch (N->getOpcode()) {
1326 default:
1327#ifndef NDEBUG
1328 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1329 N->dump(&DAG); dbgs() << "\n";
1330#endif
1331 report_fatal_error("Do not know how to expand the result of this "
1332 "operator!");
1333
1334 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1335 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1336 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1337
1338 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1339 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1340 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1341 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1342 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1343 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1344
1345 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1346 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1347 case ISD::STRICT_FMINNUM:
1348 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1349 case ISD::STRICT_FMAXNUM:
1350 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1351 case ISD::STRICT_FADD:
1352 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1353 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1354 case ISD::STRICT_FCEIL:
1355 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1356 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1357 case ISD::STRICT_FCOS:
1358 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1359 case ISD::STRICT_FDIV:
1360 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1361 case ISD::STRICT_FEXP:
1362 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1363 case ISD::STRICT_FEXP2:
1364 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1365 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1366 case ISD::STRICT_FFLOOR:
1367 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1368 case ISD::STRICT_FLOG:
1369 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1370 case ISD::STRICT_FLOG2:
1371 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1372 case ISD::STRICT_FLOG10:
1373 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1374 case ISD::STRICT_FMA:
1375 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1376 case ISD::STRICT_FMUL:
1377 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1378 case ISD::STRICT_FNEARBYINT:
1379 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1380 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1381 case ISD::STRICT_FP_EXTEND:
1382 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1383 case ISD::STRICT_FPOW:
1384 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1385 case ISD::STRICT_FPOWI:
1386 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1387 case ISD::FLDEXP:
1388 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1389 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1390 case ISD::STRICT_FRINT:
1391 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1392 case ISD::STRICT_FROUND:
1393 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1394 case ISD::STRICT_FROUNDEVEN:
1395 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1396 case ISD::STRICT_FSIN:
1397 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1398 case ISD::STRICT_FSQRT:
1399 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1400 case ISD::STRICT_FSUB:
1401 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1402 case ISD::STRICT_FTRUNC:
1403 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1404 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1405 case ISD::STRICT_SINT_TO_FP:
1406 case ISD::STRICT_UINT_TO_FP:
1407 case ISD::SINT_TO_FP:
1408 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1409 case ISD::STRICT_FREM:
1410 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1411 }
1412
1413 // If Lo/Hi is null, the sub-method took care of registering results etc.
1414 if (Lo.getNode())
1415 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1416}
1417
1418void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1419 SDValue &Hi) {
1420 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1421 assert(NVT.getSizeInBits() == 64 &&
1422 "Do not know how to expand this float constant!");
1423 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1424 SDLoc dl(N);
1425 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1426 APInt(64, C.getRawData()[1])),
1427 dl, NVT);
1428 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1429 APInt(64, C.getRawData()[0])),
1430 dl, NVT);
1431}
1432
1433void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1434 SDValue &Lo, SDValue &Hi) {
1435 bool IsStrict = N->isStrictFPOpcode();
1436 unsigned Offset = IsStrict ? 1 : 0;
1437 SDValue Op = N->getOperand(0 + Offset);
1438 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1439 TargetLowering::MakeLibCallOptions CallOptions;
1440 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1441 Op, CallOptions, SDLoc(N),
1442 Chain);
1443 if (IsStrict)
1444 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1445 GetPairElements(Tmp.first, Lo, Hi);
1446}
1447
1448void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1449 SDValue &Lo, SDValue &Hi) {
1450 bool IsStrict = N->isStrictFPOpcode();
1451 unsigned Offset = IsStrict ? 1 : 0;
1452 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1453 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1454 TargetLowering::MakeLibCallOptions CallOptions;
1455 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1456 Ops, CallOptions, SDLoc(N),
1457 Chain);
1458 if (IsStrict)
1459 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1460 GetPairElements(Tmp.first, Lo, Hi);
1461}
1462
1463void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1464 SDValue &Hi) {
1465 assert(N->getValueType(0) == MVT::ppcf128 &&
1466 "Logic only correct for ppcf128!");
1467 SDLoc dl(N);
1468 SDValue Tmp;
1469 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1470 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1471 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1472 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1473 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1474 ISD::SETEQ);
1475}
1476
1477void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1478 SDValue &Hi) {
1479 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1480 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1481 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1482 RTLIB::FMIN_PPCF128), Lo, Hi);
1483}
1484
1485void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1486 SDValue &Hi) {
1487 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1488 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1489 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1490 RTLIB::FMAX_PPCF128), Lo, Hi);
1491}
1492
1493void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1494 SDValue &Hi) {
1495 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1496 RTLIB::ADD_F32, RTLIB::ADD_F64,
1497 RTLIB::ADD_F80, RTLIB::ADD_F128,
1498 RTLIB::ADD_PPCF128), Lo, Hi);
1499}
1500
1501void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1502 SDValue &Hi) {
1503 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1504 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1505 RTLIB::CBRT_F128,
1506 RTLIB::CBRT_PPCF128), Lo, Hi);
1507}
1508
1509void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1510 SDValue &Lo, SDValue &Hi) {
1511 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1512 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1513 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1514 RTLIB::CEIL_PPCF128), Lo, Hi);
1515}
1516
1517void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1518 SDValue &Lo, SDValue &Hi) {
1519 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1520 RTLIB::COPYSIGN_F32,
1521 RTLIB::COPYSIGN_F64,
1522 RTLIB::COPYSIGN_F80,
1523 RTLIB::COPYSIGN_F128,
1524 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1525}
1526
1527void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1528 SDValue &Lo, SDValue &Hi) {
1529 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1530 RTLIB::COS_F32, RTLIB::COS_F64,
1531 RTLIB::COS_F80, RTLIB::COS_F128,
1532 RTLIB::COS_PPCF128), Lo, Hi);
1533}
1534
1535void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1536 SDValue &Hi) {
1537 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1538 RTLIB::DIV_F32,
1539 RTLIB::DIV_F64,
1540 RTLIB::DIV_F80,
1541 RTLIB::DIV_F128,
1542 RTLIB::DIV_PPCF128), Lo, Hi);
1543}
1544
1545void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1546 SDValue &Lo, SDValue &Hi) {
1547 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1548 RTLIB::EXP_F32, RTLIB::EXP_F64,
1549 RTLIB::EXP_F80, RTLIB::EXP_F128,
1550 RTLIB::EXP_PPCF128), Lo, Hi);
1551}
1552
1553void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1554 SDValue &Lo, SDValue &Hi) {
1555 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1556 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1557 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1558 RTLIB::EXP2_PPCF128), Lo, Hi);
1559}
1560
1561void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1562 SDValue &Hi) {
1563 ExpandFloatRes_Unary(N,
1564 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1565 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1566 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1567 Lo, Hi);
1568}
1569
1570void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1571 SDValue &Lo, SDValue &Hi) {
1572 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1573 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1574 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1575 RTLIB::FLOOR_PPCF128), Lo, Hi);
1576}
1577
1578void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1579 SDValue &Lo, SDValue &Hi) {
1580 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1581 RTLIB::LOG_F32, RTLIB::LOG_F64,
1582 RTLIB::LOG_F80, RTLIB::LOG_F128,
1583 RTLIB::LOG_PPCF128), Lo, Hi);
1584}
1585
1586void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1587 SDValue &Lo, SDValue &Hi) {
1588 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1589 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1590 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1591 RTLIB::LOG2_PPCF128), Lo, Hi);
1592}
1593
1594void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1595 SDValue &Lo, SDValue &Hi) {
1596 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1597 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1598 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1599 RTLIB::LOG10_PPCF128), Lo, Hi);
1600}
1601
1602void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1603 SDValue &Hi) {
1604 bool IsStrict = N->isStrictFPOpcode();
1605 unsigned Offset = IsStrict ? 1 : 0;
1606 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1607 N->getOperand(2 + Offset) };
1608 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1609 TargetLowering::MakeLibCallOptions CallOptions;
1610 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1611 RTLIB::FMA_F32,
1612 RTLIB::FMA_F64,
1613 RTLIB::FMA_F80,
1614 RTLIB::FMA_F128,
1615 RTLIB::FMA_PPCF128),
1616 N->getValueType(0), Ops, CallOptions,
1617 SDLoc(N), Chain);
1618 if (IsStrict)
1619 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1620 GetPairElements(Tmp.first, Lo, Hi);
1621}
1622
1623void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1624 SDValue &Hi) {
1625 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1626 RTLIB::MUL_F32,
1627 RTLIB::MUL_F64,
1628 RTLIB::MUL_F80,
1629 RTLIB::MUL_F128,
1630 RTLIB::MUL_PPCF128), Lo, Hi);
1631}
1632
1633void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1634 SDValue &Lo, SDValue &Hi) {
1635 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1636 RTLIB::NEARBYINT_F32,
1637 RTLIB::NEARBYINT_F64,
1638 RTLIB::NEARBYINT_F80,
1639 RTLIB::NEARBYINT_F128,
1640 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1641}
1642
1643void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1644 SDValue &Hi) {
1645 SDLoc dl(N);
1646 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1647 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1648 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1649}
1650
1651void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1652 SDValue &Hi) {
1653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1654 SDLoc dl(N);
1655 bool IsStrict = N->isStrictFPOpcode();
1656
1657 SDValue Chain;
1658 if (IsStrict) {
1659 // If the expanded type is the same as the input type, just bypass the node.
1660 if (NVT == N->getOperand(1).getValueType()) {
1661 Hi = N->getOperand(1);
1662 Chain = N->getOperand(0);
1663 } else {
1664 // Other we need to extend.
1665 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1666 { N->getOperand(0), N->getOperand(1) });
1667 Chain = Hi.getValue(1);
1668 }
1669 } else {
1670 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1671 }
1672
1673 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1674 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1675
1676 if (IsStrict)
1677 ReplaceValueWith(SDValue(N, 1), Chain);
1678}
1679
1680void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1681 SDValue &Lo, SDValue &Hi) {
1682 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1683 RTLIB::POW_F32, RTLIB::POW_F64,
1684 RTLIB::POW_F80, RTLIB::POW_F128,
1685 RTLIB::POW_PPCF128), Lo, Hi);
1686}
1687
1688void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1689 SDValue &Lo, SDValue &Hi) {
1690 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1691}
1692
1693void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1694 SDValue &Hi) {
1695 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1696}
1697
1698void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1699 SDValue &Lo, SDValue &Hi) {
1700 assert(N->getValueType(0) == MVT::ppcf128 &&
1701 "Logic only correct for ppcf128!");
1702
1703 SDLoc dl(N);
1704 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1705 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1706 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1707}
1708
1709void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1710 SDValue &Lo, SDValue &Hi) {
1711 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1712 RTLIB::REM_F32, RTLIB::REM_F64,
1713 RTLIB::REM_F80, RTLIB::REM_F128,
1714 RTLIB::REM_PPCF128), Lo, Hi);
1715}
1716
1717void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1718 SDValue &Lo, SDValue &Hi) {
1719 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1720 RTLIB::RINT_F32, RTLIB::RINT_F64,
1721 RTLIB::RINT_F80, RTLIB::RINT_F128,
1722 RTLIB::RINT_PPCF128), Lo, Hi);
1723}
1724
1725void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1726 SDValue &Lo, SDValue &Hi) {
1727 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1728 RTLIB::ROUND_F32,
1729 RTLIB::ROUND_F64,
1730 RTLIB::ROUND_F80,
1731 RTLIB::ROUND_F128,
1732 RTLIB::ROUND_PPCF128), Lo, Hi);
1733}
1734
1735void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1736 SDValue &Lo, SDValue &Hi) {
1737 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1738 RTLIB::ROUNDEVEN_F32,
1739 RTLIB::ROUNDEVEN_F64,
1740 RTLIB::ROUNDEVEN_F80,
1741 RTLIB::ROUNDEVEN_F128,
1742 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1743}
1744
1745void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1746 SDValue &Lo, SDValue &Hi) {
1747 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1748 RTLIB::SIN_F32, RTLIB::SIN_F64,
1749 RTLIB::SIN_F80, RTLIB::SIN_F128,
1750 RTLIB::SIN_PPCF128), Lo, Hi);
1751}
1752
1753void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1754 SDValue &Lo, SDValue &Hi) {
1755 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1756 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1757 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1758 RTLIB::SQRT_PPCF128), Lo, Hi);
1759}
1760
1761void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1762 SDValue &Hi) {
1763 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1764 RTLIB::SUB_F32,
1765 RTLIB::SUB_F64,
1766 RTLIB::SUB_F80,
1767 RTLIB::SUB_F128,
1768 RTLIB::SUB_PPCF128), Lo, Hi);
1769}
1770
1771void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1772 SDValue &Lo, SDValue &Hi) {
1773 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1774 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1775 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1776 RTLIB::TRUNC_PPCF128), Lo, Hi);
1777}
1778
1779void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1780 SDValue &Hi) {
1781 if (ISD::isNormalLoad(N)) {
1782 ExpandRes_NormalLoad(N, Lo, Hi);
1783 return;
1784 }
1785
1786 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1787 LoadSDNode *LD = cast<LoadSDNode>(N);
1788 SDValue Chain = LD->getChain();
1789 SDValue Ptr = LD->getBasePtr();
1790 SDLoc dl(N);
1791
1792 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1793 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1794 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1795
1796 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1797 LD->getMemoryVT(), LD->getMemOperand());
1798
1799 // Remember the chain.
1800 Chain = Hi.getValue(1);
1801
1802 // The low part is zero.
1803 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1804 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1805
1806 // Modified the chain - switch anything that used the old chain to use the
1807 // new one.
1808 ReplaceValueWith(SDValue(LD, 1), Chain);
1809}
1810
1811void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1814 EVT VT = N->getValueType(0);
1815 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1816 bool Strict = N->isStrictFPOpcode();
1817 SDValue Src = N->getOperand(Strict ? 1 : 0);
1818 EVT SrcVT = Src.getValueType();
1819 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1820 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1821 SDLoc dl(N);
1822 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1823
1824 // TODO: Any other flags to propagate?
1825 SDNodeFlags Flags;
1826 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1827
1828 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1829 // When promoting partial word types to i32 we must honor the signedness,
1830 // though.
1831 if (SrcVT.bitsLE(MVT::i32)) {
1832 // The integer can be represented exactly in an f64.
1833 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1834 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1835 if (Strict) {
1836 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1837 {Chain, Src}, Flags);
1838 Chain = Hi.getValue(1);
1839 } else
1840 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1841 } else {
1842 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1843 if (SrcVT.bitsLE(MVT::i64)) {
1844 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1845 MVT::i64, Src);
1846 LC = RTLIB::SINTTOFP_I64_PPCF128;
1847 } else if (SrcVT.bitsLE(MVT::i128)) {
1848 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1849 LC = RTLIB::SINTTOFP_I128_PPCF128;
1850 }
1851 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1852
1853 TargetLowering::MakeLibCallOptions CallOptions;
1854 CallOptions.setSExt(true);
1855 std::pair<SDValue, SDValue> Tmp =
1856 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1857 if (Strict)
1858 Chain = Tmp.second;
1859 GetPairElements(Tmp.first, Lo, Hi);
1860 }
1861
1862 // No need to complement for unsigned 32-bit integers
1863 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1864 if (Strict)
1865 ReplaceValueWith(SDValue(N, 1), Chain);
1866
1867 return;
1868 }
1869
1870 // Unsigned - fix up the SINT_TO_FP value just calculated.
1871 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1872 // keep semantics correctness if the integer is not exactly representable
1873 // here. See ExpandLegalINT_TO_FP.
1874 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1875 SrcVT = Src.getValueType();
1876
1877 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1878 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1879 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1880 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1881 ArrayRef<uint64_t> Parts;
1882
1883 switch (SrcVT.getSimpleVT().SimpleTy) {
1884 default:
1885 llvm_unreachable("Unsupported UINT_TO_FP!");
1886 case MVT::i32:
1887 Parts = TwoE32;
1888 break;
1889 case MVT::i64:
1890 Parts = TwoE64;
1891 break;
1892 case MVT::i128:
1893 Parts = TwoE128;
1894 break;
1895 }
1896
1897 // TODO: Are there other fast-math-flags to propagate to this FADD?
1898 SDValue NewLo = DAG.getConstantFP(
1899 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1900 if (Strict) {
1901 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1902 {Chain, Hi, NewLo}, Flags);
1903 Chain = Lo.getValue(1);
1904 ReplaceValueWith(SDValue(N, 1), Chain);
1905 } else
1906 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1907 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1908 Lo, Hi, ISD::SETLT);
1909 GetPairElements(Lo, Lo, Hi);
1910}
1911
1912
1913//===----------------------------------------------------------------------===//
1914// Float Operand Expansion
1915//===----------------------------------------------------------------------===//
1916
1917/// ExpandFloatOperand - This method is called when the specified operand of the
1918/// specified node is found to need expansion. At this point, all of the result
1919/// types of the node are known to be legal, but other operands of the node may
1920/// need promotion or expansion as well as the specified one.
1921bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1922 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1923 SDValue Res = SDValue();
1924
1925 // See if the target wants to custom expand this node.
1926 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1927 return false;
1928
1929 switch (N->getOpcode()) {
1930 default:
1931#ifndef NDEBUG
1932 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1933 N->dump(&DAG); dbgs() << "\n";
1934#endif
1935 report_fatal_error("Do not know how to expand this operator's operand!");
1936
1937 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1938 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1939 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1940
1941 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1942 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1943 case ISD::STRICT_FP_ROUND:
1944 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1945 case ISD::STRICT_FP_TO_SINT:
1946 case ISD::STRICT_FP_TO_UINT:
1947 case ISD::FP_TO_SINT:
1948 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1949 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1950 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1951 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1952 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1953 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1954 case ISD::STRICT_FSETCC:
1955 case ISD::STRICT_FSETCCS:
1956 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1957 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1958 OpNo); break;
1959 }
1960
1961 // If the result is null, the sub-method took care of registering results etc.
1962 if (!Res.getNode()) return false;
1963
1964 // If the result is N, the sub-method updated N in place. Tell the legalizer
1965 // core about this.
1966 if (Res.getNode() == N)
1967 return true;
1968
1969 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1970 "Invalid operand expansion");
1971
1972 ReplaceValueWith(SDValue(N, 0), Res);
1973 return false;
1974}
1975
1976/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1977/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1978void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1979 SDValue &NewRHS,
1980 ISD::CondCode &CCCode,
1981 const SDLoc &dl, SDValue &Chain,
1982 bool IsSignaling) {
1983 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1984 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1985 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1986
1987 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1988
1989 // FIXME: This generated code sucks. We want to generate
1990 // FCMPU crN, hi1, hi2
1991 // BNE crN, L:
1992 // FCMPU crN, lo1, lo2
1993 // The following can be improved, but not that much.
1994 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1995 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1996 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1997 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1998 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1999 RHSLo, CCCode, OutputChain, IsSignaling);
2000 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2001 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2002 Tmp1 =
2003 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2004 ISD::SETUNE, OutputChain, IsSignaling);
2005 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2006 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2007 RHSHi, CCCode, OutputChain, IsSignaling);
2008 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2009 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2010 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2011 NewRHS = SDValue(); // LHS is the result, not a compare.
2012 Chain = OutputChain;
2013}
2014
2015SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2016 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2017 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2018 SDValue Chain;
2019 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2020
2021 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2022 // against zero to select between true and false values.
2023 if (!NewRHS.getNode()) {
2024 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2025 CCCode = ISD::SETNE;
2026 }
2027
2028 // Update N to have the operands specified.
2029 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2030 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2031 N->getOperand(4)), 0);
2032}
2033
2034SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2035 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2036 "Logic only correct for ppcf128!");
2037 SDValue Lo, Hi;
2038 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2039 // The ppcf128 value is providing only the sign; take it from the
2040 // higher-order double (which must have the larger magnitude).
2041 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2042 N->getValueType(0), N->getOperand(0), Hi);
2043}
2044
2045SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2046 bool IsStrict = N->isStrictFPOpcode();
2047 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2048 "Logic only correct for ppcf128!");
2049 SDValue Lo, Hi;
2050 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2051
2052 if (!IsStrict)
2053 // Round it the rest of the way (e.g. to f32) if needed.
2054 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2055 N->getValueType(0), Hi, N->getOperand(1));
2056
2057 // Eliminate the node if the input float type is the same as the output float
2058 // type.
2059 if (Hi.getValueType() == N->getValueType(0)) {
2060 // Connect the output chain to the input chain, unlinking the node.
2061 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2062 ReplaceValueWith(SDValue(N, 0), Hi);
2063 return SDValue();
2064 }
2065
2066 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2067 {N->getValueType(0), MVT::Other},
2068 {N->getOperand(0), Hi, N->getOperand(2)});
2069 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2070 ReplaceValueWith(SDValue(N, 0), Expansion);
2071 return SDValue();
2072}
2073
2074SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2075 EVT RVT = N->getValueType(0);
2076 SDLoc dl(N);
2077
2078 bool IsStrict = N->isStrictFPOpcode();
2079 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2080 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2081 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2082 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2083
2084 EVT NVT;
2085 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2086 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2087 "Unsupported FP_TO_XINT!");
2088 TargetLowering::MakeLibCallOptions CallOptions;
2089 std::pair<SDValue, SDValue> Tmp =
2090 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2091 if (!IsStrict)
2092 return Tmp.first;
2093
2094 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2095 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2096 return SDValue();
2097}
2098
2099SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2100 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2101 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2102 SDValue Chain;
2103 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2104
2105 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2106 // against zero to select between true and false values.
2107 if (!NewRHS.getNode()) {
2108 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2109 CCCode = ISD::SETNE;
2110 }
2111
2112 // Update N to have the operands specified.
2113 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2114 N->getOperand(2), N->getOperand(3),
2115 DAG.getCondCode(CCCode)), 0);
2116}
2117
2118SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2119 bool IsStrict = N->isStrictFPOpcode();
2120 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2121 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2122 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2123 ISD::CondCode CCCode =
2124 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2125 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2126 N->getOpcode() == ISD::STRICT_FSETCCS);
2127
2128 // FloatExpandSetCCOperands always returned a scalar.
2129 assert(!NewRHS.getNode() && "Expect to return scalar");
2130 assert(NewLHS.getValueType() == N->getValueType(0) &&
2131 "Unexpected setcc expansion!");
2132 if (Chain) {
2133 ReplaceValueWith(SDValue(N, 0), NewLHS);
2134 ReplaceValueWith(SDValue(N, 1), Chain);
2135 return SDValue();
2136 }
2137 return NewLHS;
2138}
2139
2140SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2141 if (ISD::isNormalStore(N))
2142 return ExpandOp_NormalStore(N, OpNo);
2143
2144 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2145 assert(OpNo == 1 && "Can only expand the stored value so far");
2146 StoreSDNode *ST = cast<StoreSDNode>(N);
2147
2148 SDValue Chain = ST->getChain();
2149 SDValue Ptr = ST->getBasePtr();
2150
2151 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2152 ST->getValue().getValueType());
2153 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2154 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2155 (void)NVT;
2156
2157 SDValue Lo, Hi;
2158 GetExpandedOp(ST->getValue(), Lo, Hi);
2159
2160 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2161 ST->getMemoryVT(), ST->getMemOperand());
2162}
2163
2164SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2165 EVT RVT = N->getValueType(0);
2166 EVT RetVT = N->getOperand(0).getValueType();
2167 TargetLowering::MakeLibCallOptions CallOptions;
2168 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2169 RTLIB::LROUND_F32,
2170 RTLIB::LROUND_F64,
2171 RTLIB::LROUND_F80,
2172 RTLIB::LROUND_F128,
2173 RTLIB::LROUND_PPCF128),
2174 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2175}
2176
2177SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2178 EVT RVT = N->getValueType(0);
2179 EVT RetVT = N->getOperand(0).getValueType();
2180 TargetLowering::MakeLibCallOptions CallOptions;
2181 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2182 RTLIB::LLROUND_F32,
2183 RTLIB::LLROUND_F64,
2184 RTLIB::LLROUND_F80,
2185 RTLIB::LLROUND_F128,
2186 RTLIB::LLROUND_PPCF128),
2187 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2188}
2189
2190SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2191 EVT RVT = N->getValueType(0);
2192 EVT RetVT = N->getOperand(0).getValueType();
2193 TargetLowering::MakeLibCallOptions CallOptions;
2194 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2195 RTLIB::LRINT_F32,
2196 RTLIB::LRINT_F64,
2197 RTLIB::LRINT_F80,
2198 RTLIB::LRINT_F128,
2199 RTLIB::LRINT_PPCF128),
2200 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2201}
2202
2203SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2204 EVT RVT = N->getValueType(0);
2205 EVT RetVT = N->getOperand(0).getValueType();
2206 TargetLowering::MakeLibCallOptions CallOptions;
2207 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2208 RTLIB::LLRINT_F32,
2209 RTLIB::LLRINT_F64,
2210 RTLIB::LLRINT_F80,
2211 RTLIB::LLRINT_F128,
2212 RTLIB::LLRINT_PPCF128),
2213 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2214}
2215
2216//===----------------------------------------------------------------------===//
2217// Float Operand Promotion
2218//===----------------------------------------------------------------------===//
2219//
2220
2221static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2222 if (OpVT == MVT::f16) {
2223 return ISD::FP16_TO_FP;
2224 } else if (RetVT == MVT::f16) {
2225 return ISD::FP_TO_FP16;
2226 } else if (OpVT == MVT::bf16) {
2227 return ISD::BF16_TO_FP;
2228 } else if (RetVT == MVT::bf16) {
2229 return ISD::FP_TO_BF16;
2230 }
2231
2232 report_fatal_error("Attempt at an invalid promotion-related conversion");
2233}
2234
2235static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2236 if (OpVT == MVT::f16)
2237 return ISD::STRICT_FP16_TO_FP;
2238
2239 if (RetVT == MVT::f16)
2240 return ISD::STRICT_FP_TO_FP16;
2241
2242 if (OpVT == MVT::bf16)
2243 return ISD::STRICT_BF16_TO_FP;
2244
2245 if (RetVT == MVT::bf16)
2246 return ISD::STRICT_FP_TO_BF16;
2247
2248 report_fatal_error("Attempt at an invalid promotion-related conversion");
2249}
2250
2251bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2252 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2253 SDValue R = SDValue();
2254
2255 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2256 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2257 return false;
2258 }
2259
2260 // Nodes that use a promotion-requiring floating point operand, but doesn't
2261 // produce a promotion-requiring floating point result, need to be legalized
2262 // to use the promoted float operand. Nodes that produce at least one
2263 // promotion-requiring floating point result have their operands legalized as
2264 // a part of PromoteFloatResult.
2265 // clang-format off
2266 switch (N->getOpcode()) {
2267 default:
2268 #ifndef NDEBUG
2269 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2270 N->dump(&DAG); dbgs() << "\n";
2271 #endif
2272 report_fatal_error("Do not know how to promote this operator's operand!");
2273
2274 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2275 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2276 case ISD::FP_TO_SINT:
2277 case ISD::FP_TO_UINT:
2278 case ISD::LRINT:
2279 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2280 case ISD::FP_TO_SINT_SAT:
2281 case ISD::FP_TO_UINT_SAT:
2282 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2283 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2284 case ISD::STRICT_FP_EXTEND:
2285 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2286 break;
2287 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2288 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2289 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2290 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2291 }
2292 // clang-format on
2293
2294 if (R.getNode())
2295 ReplaceValueWith(SDValue(N, 0), R);
2296 return false;
2297}
2298
2299SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2300 SDValue Op = N->getOperand(0);
2301 EVT OpVT = Op->getValueType(0);
2302
2303 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2304 EVT PromotedVT = Promoted->getValueType(0);
2305
2306 // Convert the promoted float value to the desired IVT.
2307 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2308 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2309 IVT, Promoted);
2310 // The final result type might not be an scalar so we need a bitcast. The
2311 // bitcast will be further legalized if needed.
2312 return DAG.getBitcast(N->getValueType(0), Convert);
2313}
2314
2315// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2316// PromoteFloatRes_FCOPYSIGN.
2317SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2318 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2319 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2320
2321 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2322 N->getOperand(0), Op1);
2323}
2324
2325// Convert the promoted float value to the desired integer type
2326SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2327 SDValue Op = GetPromotedFloat(N->getOperand(0));
2328 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2329}
2330
2331SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2332 unsigned OpNo) {
2333 SDValue Op = GetPromotedFloat(N->getOperand(0));
2334 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2335 N->getOperand(1));
2336}
2337
2338SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2339 SDValue Op = GetPromotedFloat(N->getOperand(0));
2340 EVT VT = N->getValueType(0);
2341
2342 // Desired VT is same as promoted type. Use promoted float directly.
2343 if (VT == Op->getValueType(0))
2344 return Op;
2345
2346 // Else, extend the promoted float value to the desired VT.
2347 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2348}
2349
2350SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2351 unsigned OpNo) {
2352 assert(OpNo == 1 && "Promoting unpromotable operand");
2353
2354 SDValue Op = GetPromotedFloat(N->getOperand(1));
2355 EVT VT = N->getValueType(0);
2356
2357 // Desired VT is same as promoted type. Use promoted float directly.
2358 if (VT == Op->getValueType(0)) {
2359 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2360 return Op;
2361 }
2362
2363 // Else, extend the promoted float value to the desired VT.
2364 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2365 N->getOperand(0), Op);
2366 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2367 return Res;
2368}
2369
2370// Promote the float operands used for comparison. The true- and false-
2371// operands have the same type as the result and are promoted, if needed, by
2372// PromoteFloatRes_SELECT_CC
2373SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2374 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2375 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2376
2377 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2378 LHS, RHS, N->getOperand(2), N->getOperand(3),
2379 N->getOperand(4));
2380}
2381
2382// Construct a SETCC that compares the promoted values and sets the conditional
2383// code.
2384SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2385 EVT VT = N->getValueType(0);
2386 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2387 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2388 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2389
2390 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2391
2392}
2393
2394// Lower the promoted Float down to the integer value of same size and construct
2395// a STORE of the integer value.
2396SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2397 StoreSDNode *ST = cast<StoreSDNode>(N);
2398 SDValue Val = ST->getValue();
2399 SDLoc DL(N);
2400
2401 SDValue Promoted = GetPromotedFloat(Val);
2402 EVT VT = ST->getOperand(1).getValueType();
2403 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2404
2405 SDValue NewVal;
2406 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2407 IVT, Promoted);
2408
2409 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2410 ST->getMemOperand());
2411}
2412
2413SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2414 unsigned OpNo) {
2415 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2416 SDValue Val = ST->getVal();
2417 SDLoc DL(N);
2418
2419 SDValue Promoted = GetPromotedFloat(Val);
2420 EVT VT = ST->getOperand(1).getValueType();
2421 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2422
2423 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2424 DL, IVT, Promoted);
2425
2426 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2427 ST->getBasePtr(), ST->getMemOperand());
2428}
2429
2430//===----------------------------------------------------------------------===//
2431// Float Result Promotion
2432//===----------------------------------------------------------------------===//
2433
2434void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2435 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2436 SDValue R = SDValue();
2437
2438 // See if the target wants to custom expand this node.
2439 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2440 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2441 return;
2442 }
2443
2444 switch (N->getOpcode()) {
2445 // These opcodes cannot appear if promotion of FP16 is done in the backend
2446 // instead of Clang
2447 case ISD::FP16_TO_FP:
2448 case ISD::FP_TO_FP16:
2449 default:
2450#ifndef NDEBUG
2451 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2452 N->dump(&DAG); dbgs() << "\n";
2453#endif
2454 report_fatal_error("Do not know how to promote this operator's result!");
2455
2456 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2457 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2458 case ISD::EXTRACT_VECTOR_ELT:
2459 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2460 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2461
2462 // Unary FP Operations
2463 case ISD::FABS:
2464 case ISD::FCBRT:
2465 case ISD::FCEIL:
2466 case ISD::FCOS:
2467 case ISD::FEXP:
2468 case ISD::FEXP2:
2469 case ISD::FEXP10:
2470 case ISD::FFLOOR:
2471 case ISD::FLOG:
2472 case ISD::FLOG2:
2473 case ISD::FLOG10:
2474 case ISD::FNEARBYINT:
2475 case ISD::FNEG:
2476 case ISD::FRINT:
2477 case ISD::FROUND:
2478 case ISD::FROUNDEVEN:
2479 case ISD::FSIN:
2480 case ISD::FSQRT:
2481 case ISD::FTRUNC:
2482 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2483
2484 // Binary FP Operations
2485 case ISD::FADD:
2486 case ISD::FDIV:
2487 case ISD::FMAXIMUM:
2488 case ISD::FMINIMUM:
2489 case ISD::FMAXNUM:
2490 case ISD::FMINNUM:
2491 case ISD::FMUL:
2492 case ISD::FPOW:
2493 case ISD::FREM:
2494 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2495
2496 case ISD::FMA: // FMA is same as FMAD
2497 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2498
2499 case ISD::FPOWI:
2500 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2501 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2502
2503 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2504 case ISD::STRICT_FP_ROUND:
2505 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2506 break;
2507 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2508 case ISD::ATOMIC_LOAD:
2509 R = PromoteFloatRes_ATOMIC_LOAD(N);
2510 break;
2511 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2512 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2513
2514 case ISD::SINT_TO_FP:
2515 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2516 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2517 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2518 case ISD::VECREDUCE_FADD:
2519 case ISD::VECREDUCE_FMUL:
2520 case ISD::VECREDUCE_FMIN:
2521 case ISD::VECREDUCE_FMAX:
2522 case ISD::VECREDUCE_FMAXIMUM:
2523 case ISD::VECREDUCE_FMINIMUM:
2524 R = PromoteFloatRes_VECREDUCE(N);
2525 break;
2526 case ISD::VECREDUCE_SEQ_FADD:
2527 case ISD::VECREDUCE_SEQ_FMUL:
2528 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2529 break;
2530 }
2531
2532 if (R.getNode())
2533 SetPromotedFloat(SDValue(N, ResNo), R);
2534}
2535
2536// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2537// At this point, it is not possible to determine if the bitcast value is
2538// eventually stored to memory or promoted to f32 or promoted to a floating
2539// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2540// STORE promotion handlers.
2541SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2542 EVT VT = N->getValueType(0);
2543 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2544 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2545 // bitcast will be legalized further if necessary.
2546 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2547 N->getOperand(0).getValueType().getSizeInBits());
2548 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2549 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2550}
2551
2552SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2553 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2554 EVT VT = N->getValueType(0);
2555 SDLoc DL(N);
2556
2557 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2558 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2559 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2560 IVT);
2561
2562 // Convert the Constant to the desired FP type
2563 // FIXME We might be able to do the conversion during compilation and get rid
2564 // of it from the object code
2565 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2566 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2567}
2568
2569// If the Index operand is a constant, try to redirect the extract operation to
2570// the correct legalized vector. If not, bit-convert the input vector to
2571// equivalent integer vector. Extract the element as an (bit-cast) integer
2572// value and convert it to the promoted type.
2573SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2574 SDLoc DL(N);
2575
2576 // If the index is constant, try to extract the value from the legalized
2577 // vector type.
2578 if (isa<ConstantSDNode>(N->getOperand(1))) {
2579 SDValue Vec = N->getOperand(0);
2580 SDValue Idx = N->getOperand(1);
2581 EVT VecVT = Vec->getValueType(0);
2582 EVT EltVT = VecVT.getVectorElementType();
2583
2584 uint64_t IdxVal = Idx->getAsZExtVal();
2585
2586 switch (getTypeAction(VecVT)) {
2587 default: break;
2588 case TargetLowering::TypeScalarizeVector: {
2589 SDValue Res = GetScalarizedVector(N->getOperand(0));
2590 ReplaceValueWith(SDValue(N, 0), Res);
2591 return SDValue();
2592 }
2593 case TargetLowering::TypeWidenVector: {
2594 Vec = GetWidenedVector(Vec);
2595 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2596 ReplaceValueWith(SDValue(N, 0), Res);
2597 return SDValue();
2598 }
2599 case TargetLowering::TypeSplitVector: {
2600 SDValue Lo, Hi;
2601 GetSplitVector(Vec, Lo, Hi);
2602
2603 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2604 SDValue Res;
2605 if (IdxVal < LoElts)
2606 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2607 else
2608 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2609 DAG.getConstant(IdxVal - LoElts, DL,
2610 Idx.getValueType()));
2611 ReplaceValueWith(SDValue(N, 0), Res);
2612 return SDValue();
2613 }
2614
2615 }
2616 }
2617
2618 // Bit-convert the input vector to the equivalent integer vector
2619 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2620 EVT IVT = NewOp.getValueType().getVectorElementType();
2621
2622 // Extract the element as an (bit-cast) integer value
2623 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2624 NewOp, N->getOperand(1));
2625
2626 // Convert the element to the desired FP type
2627 EVT VT = N->getValueType(0);
2628 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2629 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2630}
2631
2632// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2633// needs promotion, so does the argument X. Note that Y, if needed, will be
2634// handled during operand promotion.
2635SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2636 EVT VT = N->getValueType(0);
2637 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2638 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2639
2640 SDValue Op1 = N->getOperand(1);
2641
2642 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2643}
2644
2645// Unary operation where the result and the operand have PromoteFloat type
2646// action. Construct a new SDNode with the promoted float value of the old
2647// operand.
2648SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2649 EVT VT = N->getValueType(0);
2650 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2651 SDValue Op = GetPromotedFloat(N->getOperand(0));
2652
2653 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2654}
2655
2656// Binary operations where the result and both operands have PromoteFloat type
2657// action. Construct a new SDNode with the promoted float values of the old
2658// operands.
2659SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2660 EVT VT = N->getValueType(0);
2661 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2662 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2663 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2664 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2665}
2666
2667SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2668 EVT VT = N->getValueType(0);
2669 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2670 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2671 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2672 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2673
2674 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2675}
2676
2677// Promote the Float (first) operand and retain the Integer (second) operand
2678SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2679 EVT VT = N->getValueType(0);
2680 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2681 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2682 SDValue Op1 = N->getOperand(1);
2683
2684 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2685}
2686
2687SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2688 EVT VT = N->getValueType(0);
2689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2690 SDValue Op = GetPromotedFloat(N->getOperand(0));
2691 SDValue Res =
2692 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2693
2694 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2695 return Res;
2696}
2697
2698// Explicit operation to reduce precision. Reduce the value to half precision
2699// and promote it back to the legal type.
2700SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2701 SDLoc DL(N);
2702
2703 SDValue Op = N->getOperand(0);
2704 EVT VT = N->getValueType(0);
2705 EVT OpVT = Op->getValueType(0);
2706 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2707 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2708
2709 // Round promoted float to desired precision
2710 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2711 // Promote it back to the legal output type
2712 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2713}
2714
2715// Explicit operation to reduce precision. Reduce the value to half precision
2716// and promote it back to the legal type.
2717SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2718 SDLoc DL(N);
2719
2720 SDValue Chain = N->getOperand(0);
2721 SDValue Op = N->getOperand(1);
2722 EVT VT = N->getValueType(0);
2723 EVT OpVT = Op->getValueType(0);
2724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2725 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2726
2727 // Round promoted float to desired precision
2728 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2729 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2730 // Promote it back to the legal output type
2731 SDValue Res =
2732 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2733 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2734 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2735 return Res;
2736}
2737
2738SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2739 LoadSDNode *L = cast<LoadSDNode>(N);
2740 EVT VT = N->getValueType(0);
2741
2742 // Load the value as an integer value with the same number of bits.
2743 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2744 SDValue newL = DAG.getLoad(
2745 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2746 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2747 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2748 // Legalize the chain result by replacing uses of the old value chain with the
2749 // new one
2750 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2751
2752 // Convert the integer value to the desired FP type
2753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2754 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2755}
2756
2757SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2758 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2759 EVT VT = AM->getValueType(0);
2760
2761 // Load the value as an integer value with the same number of bits.
2762 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2763 SDValue newL = DAG.getAtomic(
2764 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
2765 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
2766
2767 // Legalize the chain result by replacing uses of the old value chain with the
2768 // new one
2769 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2770
2771 // Convert the integer value to the desired FP type
2772 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2773 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
2774}
2775
2776// Construct a new SELECT node with the promoted true- and false- values.
2777SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2778 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2779 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2780
2781 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2782 N->getOperand(0), TrueVal, FalseVal);
2783}
2784
2785// Construct a new SELECT_CC node with the promoted true- and false- values.
2786// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2787SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2788 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2789 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2790
2791 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2792 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2793 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2794}
2795
2796// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2797// float type.
2798SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2799 SDLoc DL(N);
2800 EVT VT = N->getValueType(0);
2801 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2802 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2803 // Round the value to the desired precision (that of the source type).
2804 return DAG.getNode(
2805 ISD::FP_EXTEND, DL, NVT,
2806 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2807 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2808}
2809
2810SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2811 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2812 N->getValueType(0)));
2813}
2814
2815SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2816 // Expand and promote recursively.
2817 // TODO: This is non-optimal, but dealing with the concurrently happening
2818 // vector-legalization is non-trivial. We could do something similar to
2819 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2820 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2821 return SDValue();
2822}
2823
2824SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2825 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2826 return SDValue();
2827}
2828
2829SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2830 EVT VT = N->getValueType(0);
2831
2832 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2833 SDLoc SL(N);
2834
2835 SDValue CastVal = BitConvertToInteger(AM->getVal());
2836 EVT CastVT = CastVal.getValueType();
2837
2838 SDValue NewAtomic
2839 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2840 DAG.getVTList(CastVT, MVT::Other),
2841 { AM->getChain(), AM->getBasePtr(), CastVal },
2842 AM->getMemOperand());
2843
2844 SDValue Result = NewAtomic;
2845
2846 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2847 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2848 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2849 NewAtomic);
2850 }
2851
2852 // Legalize the chain result by replacing uses of the old value chain with the
2853 // new one
2854 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2855
2856 return Result;
2857
2858}
2859
2860//===----------------------------------------------------------------------===//
2861// Half Result Soft Promotion
2862//===----------------------------------------------------------------------===//
2863
2864void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2865 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2866 N->dump(&DAG));
2867 SDValue R = SDValue();
2868
2869 // See if the target wants to custom expand this node.
2870 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2871 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2872 return;
2873 }
2874
2875 switch (N->getOpcode()) {
2876 default:
2877#ifndef NDEBUG
2878 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2879 N->dump(&DAG); dbgs() << "\n";
2880#endif
2881 report_fatal_error("Do not know how to soft promote this operator's "
2882 "result!");
2883
2884 case ISD::ARITH_FENCE:
2885 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2886 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2887 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2888 case ISD::EXTRACT_VECTOR_ELT:
2889 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2890 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2891 case ISD::STRICT_FP_ROUND:
2892 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2893
2894 // Unary FP Operations
2895 case ISD::FABS:
2896 case ISD::FCBRT:
2897 case ISD::FCEIL:
2898 case ISD::FCOS:
2899 case ISD::FEXP:
2900 case ISD::FEXP2:
2901 case ISD::FEXP10:
2902 case ISD::FFLOOR:
2903 case ISD::FLOG:
2904 case ISD::FLOG2:
2905 case ISD::FLOG10:
2906 case ISD::FNEARBYINT:
2907 case ISD::FNEG:
2908 case ISD::FREEZE:
2909 case ISD::FRINT:
2910 case ISD::FROUND:
2911 case ISD::FROUNDEVEN:
2912 case ISD::FSIN:
2913 case ISD::FSQRT:
2914 case ISD::FTRUNC:
2915 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2916
2917 // Binary FP Operations
2918 case ISD::FADD:
2919 case ISD::FDIV:
2920 case ISD::FMAXIMUM:
2921 case ISD::FMINIMUM:
2922 case ISD::FMAXNUM:
2923 case ISD::FMINNUM:
2924 case ISD::FMUL:
2925 case ISD::FPOW:
2926 case ISD::FREM:
2927 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2928
2929 case ISD::FMA: // FMA is same as FMAD
2930 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2931
2932 case ISD::FPOWI:
2933 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2934
2935 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2936
2937 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2938 case ISD::ATOMIC_LOAD:
2939 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2940 break;
2941 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2942 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2943 case ISD::SINT_TO_FP:
2944 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2945 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2946 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2947 case ISD::VECREDUCE_FADD:
2948 case ISD::VECREDUCE_FMUL:
2949 case ISD::VECREDUCE_FMIN:
2950 case ISD::VECREDUCE_FMAX:
2951 case ISD::VECREDUCE_FMAXIMUM:
2952 case ISD::VECREDUCE_FMINIMUM:
2953 R = SoftPromoteHalfRes_VECREDUCE(N);
2954 break;
2955 case ISD::VECREDUCE_SEQ_FADD:
2956 case ISD::VECREDUCE_SEQ_FMUL:
2957 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2958 break;
2959 }
2960
2961 if (R.getNode())
2962 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2963}
2964
2965SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2966 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
2967 BitConvertToInteger(N->getOperand(0)));
2968}
2969
2970SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2971 return BitConvertToInteger(N->getOperand(0));
2972}
2973
2974SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2975 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2976
2977 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2978 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2979 MVT::i16);
2980}
2981
2982SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2983 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2984 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2985 NewOp.getValueType().getVectorElementType(), NewOp,
2986 N->getOperand(1));
2987}
2988
2989SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2990 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2991 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2992 SDLoc dl(N);
2993
2994 EVT LVT = LHS.getValueType();
2995 EVT RVT = RHS.getValueType();
2996
2997 unsigned LSize = LVT.getSizeInBits();
2998 unsigned RSize = RVT.getSizeInBits();
2999
3000 // First get the sign bit of second operand.
3001 SDValue SignBit = DAG.getNode(
3002 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3003 DAG.getConstant(RSize - 1, dl,
3004 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3005 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3006
3007 // Shift right or sign-extend it if the two operands have different types.
3008 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3009 if (SizeDiff > 0) {
3010 SignBit =
3011 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3012 DAG.getConstant(SizeDiff, dl,
3013 TLI.getShiftAmountTy(SignBit.getValueType(),
3014 DAG.getDataLayout())));
3015 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3016 } else if (SizeDiff < 0) {
3017 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3018 SignBit =
3019 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3020 DAG.getConstant(-SizeDiff, dl,
3021 TLI.getShiftAmountTy(SignBit.getValueType(),
3022 DAG.getDataLayout())));
3023 }
3024
3025 // Clear the sign bit of the first operand.
3026 SDValue Mask = DAG.getNode(
3027 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3028 DAG.getConstant(LSize - 1, dl,
3029 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3030 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3031 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3032
3033 // Or the value with the sign bit.
3034 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3035}
3036
3037SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3038 EVT OVT = N->getValueType(0);
3039 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3040 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3041 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3042 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3043 SDLoc dl(N);
3044
3045 // Promote to the larger FP type.
3046 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3047 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3048 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3049 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3050
3051 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3052
3053 // Convert back to FP16 as an integer.
3054 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3058 EVT OVT = N->getValueType(0);
3059 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3060 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3061 SDValue Op1 = N->getOperand(1);
3062 SDLoc dl(N);
3063
3064 // Promote to the larger FP type.
3065 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3066
3067 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3068
3069 // Convert back to FP16 as an integer.
3070 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3071}
3072
3073SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3074 EVT OVT = N->getValueType(0);
3075 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3076 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3077 SDLoc dl(N);
3078
3079 // Promote to the larger FP type.
3080 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3081
3082 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3083 DAG.getVTList(NVT, N->getValueType(1)), Op);
3084
3085 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3086
3087 // Convert back to FP16 as an integer.
3088 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3089}
3090
3091SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3092 EVT RVT = N->getValueType(0);
3093 EVT SVT = N->getOperand(0).getValueType();
3094
3095 if (N->isStrictFPOpcode()) {
3096 // FIXME: assume we only have two f16 variants for now.
3097 unsigned Opcode;
3098 if (RVT == MVT::f16)
3099 Opcode = ISD::STRICT_FP_TO_FP16;
3100 else if (RVT == MVT::bf16)
3101 Opcode = ISD::STRICT_FP_TO_BF16;
3102 else
3103 llvm_unreachable("unknown half type");
3104 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3105 {N->getOperand(0), N->getOperand(1)});
3106 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3107 return Res;
3108 }
3109
3110 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3111 N->getOperand(0));
3112}
3113
3114SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3115 LoadSDNode *L = cast<LoadSDNode>(N);
3116
3117 // Load the value as an integer value with the same number of bits.
3118 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3119 SDValue NewL =
3120 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3121 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3122 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3123 L->getMemOperand()->getFlags(), L->getAAInfo());
3124 // Legalize the chain result by replacing uses of the old value chain with the
3125 // new one
3126 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3127 return NewL;
3128}
3129
3130SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3131 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3132
3133 // Load the value as an integer value with the same number of bits.
3134 SDValue NewL = DAG.getAtomic(
3135 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3136 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3137
3138 // Legalize the chain result by replacing uses of the old value chain with the
3139 // new one
3140 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3141 return NewL;
3142}
3143
3144SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3145 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3146 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3147 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3148 Op2);
3149}
3150
3151SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3152 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3153 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3154 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3155 N->getOperand(0), N->getOperand(1), Op2, Op3,
3156 N->getOperand(4));
3157}
3158
3159SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3160 EVT OVT = N->getValueType(0);
3161 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3162 SDLoc dl(N);
3163
3164 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3165
3166 // Round the value to the softened type.
3167 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3168}
3169
3170SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3171 return DAG.getUNDEF(MVT::i16);
3172}
3173
3174SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3175 EVT OVT = N->getValueType(0);
3176 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3177 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3178 SDLoc dl(N);
3179
3180 // Promote to the larger FP type.
3181 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3182
3183 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3184
3185 // Convert back to FP16 as an integer.
3186 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3187}
3188
3189SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3190 EVT OVT = N->getValueType(0);
3191 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3192 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3193 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3194 SDLoc dl(N);
3195
3196 // Promote to the larger FP type.
3197 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3198 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3199 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3200
3201 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3202
3203 // Convert back to FP16 as an integer.
3204 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3205}
3206
3207SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3208 // Expand and soften recursively.
3209 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3210 return SDValue();
3211}
3212
3213SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3214 // Expand and soften.
3215 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3216 return SDValue();
3217}
3218
3219//===----------------------------------------------------------------------===//
3220// Half Operand Soft Promotion
3221//===----------------------------------------------------------------------===//
3222
3223bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3224 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3225 N->dump(&DAG));
3226 SDValue Res = SDValue();
3227
3228 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3229 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3230 return false;
3231 }
3232
3233 // Nodes that use a promotion-requiring floating point operand, but doesn't
3234 // produce a soft promotion-requiring floating point result, need to be
3235 // legalized to use the soft promoted float operand. Nodes that produce at
3236 // least one soft promotion-requiring floating point result have their
3237 // operands legalized as a part of PromoteFloatResult.
3238 switch (N->getOpcode()) {
3239 default:
3240 #ifndef NDEBUG
3241 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3242 N->dump(&DAG); dbgs() << "\n";
3243 #endif
3244 report_fatal_error("Do not know how to soft promote this operator's "
3245 "operand!");
3246
3247 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3248 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3249 case ISD::FP_TO_SINT:
3250 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3251 case ISD::FP_TO_SINT_SAT:
3252 case ISD::FP_TO_UINT_SAT:
3253 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3254 case ISD::STRICT_FP_EXTEND:
3255 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3256 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3257 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3258 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3259 case ISD::ATOMIC_STORE:
3260 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3261 break;
3262 case ISD::STACKMAP:
3263 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3264 break;
3265 case ISD::PATCHPOINT:
3266 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3267 break;
3268 }
3269
3270 if (!Res.getNode())
3271 return false;
3272
3273 assert(Res.getNode() != N && "Expected a new node!");
3274
3275 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3276 "Invalid operand expansion");
3277
3278 ReplaceValueWith(SDValue(N, 0), Res);
3279 return false;
3280}
3281
3282SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3283 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3284
3285 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3286}
3287
3288SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3289 unsigned OpNo) {
3290 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3291 SDValue Op1 = N->getOperand(1);
3292 EVT RVT = Op1.getValueType();
3293 SDLoc dl(N);
3294
3295 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3296
3297 Op1 = GetSoftPromotedHalf(Op1);
3298 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3299
3300 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3301 Op1);
3302}
3303
3304SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3305 EVT RVT = N->getValueType(0);
3306 bool IsStrict = N->isStrictFPOpcode();
3307 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3308 EVT SVT = Op.getValueType();
3309 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3310
3311 if (IsStrict) {
3312 unsigned Opcode;
3313 if (SVT == MVT::f16)
3314 Opcode = ISD::STRICT_FP16_TO_FP;
3315 else if (SVT == MVT::bf16)
3316 Opcode = ISD::STRICT_BF16_TO_FP;
3317 else
3318 llvm_unreachable("unknown half type");
3319 SDValue Res =
3320 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3321 {N->getOperand(0), Op});
3322 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3323 ReplaceValueWith(SDValue(N, 0), Res);
3324 return SDValue();
3325 }
3326
3327 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3328}
3329
3330SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3331 EVT RVT = N->getValueType(0);
3332 SDValue Op = N->getOperand(0);
3333 EVT SVT = Op.getValueType();
3334 SDLoc dl(N);
3335
3336 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3337
3338 Op = GetSoftPromotedHalf(Op);
3339
3340 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3341
3342 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3343}
3344
3345SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3346 EVT RVT = N->getValueType(0);
3347 SDValue Op = N->getOperand(0);
3348 EVT SVT = Op.getValueType();
3349 SDLoc dl(N);
3350
3351 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3352
3353 Op = GetSoftPromotedHalf(Op);
3354
3355 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3356
3357 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3358 N->getOperand(1));
3359}
3360
3361SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3362 unsigned OpNo) {
3363 assert(OpNo == 0 && "Can only soften the comparison values");
3364 SDValue Op0 = N->getOperand(0);
3365 SDValue Op1 = N->getOperand(1);
3366 SDLoc dl(N);
3367
3368 EVT SVT = Op0.getValueType();
3369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3370
3371 Op0 = GetSoftPromotedHalf(Op0);
3372 Op1 = GetSoftPromotedHalf(Op1);
3373
3374 // Promote to the larger FP type.
3375 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3376 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3377 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3378
3379 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3380 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3381}
3382
3383SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3384 SDValue Op0 = N->getOperand(0);
3385 SDValue Op1 = N->getOperand(1);
3386 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3387 SDLoc dl(N);
3388
3389 EVT SVT = Op0.getValueType();
3390 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3391
3392 Op0 = GetSoftPromotedHalf(Op0);
3393 Op1 = GetSoftPromotedHalf(Op1);
3394
3395 // Promote to the larger FP type.
3396 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3397 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3398 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3399
3400 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3401}
3402
3403SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3404 assert(OpNo == 1 && "Can only soften the stored value!");
3405 StoreSDNode *ST = cast<StoreSDNode>(N);
3406 SDValue Val = ST->getValue();
3407 SDLoc dl(N);
3408
3409 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3410 SDValue Promoted = GetSoftPromotedHalf(Val);
3411 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3412 ST->getMemOperand());
3413}
3414
3415SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3416 unsigned OpNo) {
3417 assert(OpNo == 1 && "Can only soften the stored value!");
3418 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3419 SDValue Val = ST->getVal();
3420 SDLoc dl(N);
3421
3422 SDValue Promoted = GetSoftPromotedHalf(Val);
3423 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3424 ST->getChain(), Promoted, ST->getBasePtr(),
3425 ST->getMemOperand());
3426}
3427
3428SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3429 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3430 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3431 SDValue Op = N->getOperand(OpNo);
3432 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3433 SDValue NewNode =
3434 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3435
3436 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3437 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3438
3439 return SDValue(); // Signal that we replaced the node ourselves.
3440}
3441
3442SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3443 unsigned OpNo) {
3444 assert(OpNo >= 7);
3445 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3446 SDValue Op = N->getOperand(OpNo);
3447 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3448 SDValue NewNode =
3449 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3450
3451 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3452 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3453
3454 return SDValue(); // Signal that we replaced the node ourselves.
3455}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:74
Expansion
DXIL Intrinsic Expansion
Definition: DXILIntrinsicExpansion.cpp:323
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:354
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:1063
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2221
GetPromotionOpcodeStrict
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2235
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:76
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1479
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1501
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1701
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:239
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:286
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2410
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:58
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:144
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:146
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1391
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1136
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:477
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:1013
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:1016
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1171
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8771
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1237
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10157
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1208
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1789
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8754
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8475
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1099
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2349
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1602
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9774
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8855
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8804
llvm::SelectionDAG::EVTToAPFloatSemantics
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Definition: SelectionDAG.h:1799
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1247
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1728
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9780
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:1995
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
Definition: SelectionDAG.cpp:1733
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2468
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10247
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2438
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:566
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:1017
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1136
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3423
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:290
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:145
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:10866
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:10917
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:10826
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:8308
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:751
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:237
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:477
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1346
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1248
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:427
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:488
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1038
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:784
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:484
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:791
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1362
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:391
llvm::ISD::VECREDUCE_FMAXIMUM
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1366
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:478
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:914
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:422
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:904
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:230
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:940
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:412
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:775
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:451
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1359
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1363
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:432
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:931
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1084
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:501
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:728
llvm::ISD::STRICT_FPOWI
@ STRICT_FPOWI
Definition: ISDOpcodes.h:414
llvm::ISD::ATOMIC_LOAD
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1244
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:212
llvm::ISD::EXTRACT_ELEMENT
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:223
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:431
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1232
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:917
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:916
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:706
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:425
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:536
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:781
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:857
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:426
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:743
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:972
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:434
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:889
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:429
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:450
llvm::ISD::STRICT_BF16_TO_FP
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:925
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:428
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:430
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:923
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:444
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:466
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:443
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:991
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:837
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:471
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:681
llvm::ISD::STRICT_FP_TO_BF16
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:926
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1360
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:401
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:421
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:435
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:419
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1269
llvm::ISD::FFREXP
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:945
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:870
llvm::ISD::STRICT_FLDEXP
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:415
llvm::ISD::STRICT_LLROUND
@ STRICT_LLROUND
Definition: ISDOpcodes.h:433
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:424
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:856
llvm::ISD::VECREDUCE_FMINIMUM
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1367
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:787
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1153
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1347
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:494
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:423
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:516
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3166
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3159
llvm::ISD::isUNINDEXEDStore
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
Definition: SelectionDAGNodes.h:3173
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1530
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3128
llvm::RTLIB::getPOWI
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:551
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:461
llvm::RTLIB::getLDEXP
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:556
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:507
llvm::RTLIB::getFREXP
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:561
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:412
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:363
llvm::RTLIB::getFPEXT
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:285
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:320
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::tgtok::FalseVal
@ FalseVal
Definition: TGLexer.h:59
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Offset
@ Offset
Definition: DWP.cpp:456
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
llvm::Op
DWARFExpression::Operation Op
Definition: DWARFExpression.cpp:22
raw_ostream.h
N
#define N
llvm::APFloatBase::PPCDoubleDouble
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::EVT::bitsGE
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1062
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::TargetLowering::MakeLibCallOptions
This structure is used to pass arguments to makeLibCall function.
Definition: TargetLowering.h:4657
llvm::TargetLowering::MakeLibCallOptions::setSExt
MakeLibCallOptions & setSExt(bool Value=true)
Definition: TargetLowering.h:4672
llvm::TargetLowering::MakeLibCallOptions::setTypeListBeforeSoften
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
Definition: TargetLowering.h:4692
Generated on Tue May 7 2024 22:18:50 for LLVM by doxygen 1.9.6