LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
135 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
136 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
137 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
138 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
139 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
140 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
143 case ISD::SINT_TO_FP:
144 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
145 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
146 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
152 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
154 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
155 // clang-format on
156 }
157
158 // If R is null, the sub-method took care of registering the result.
159 if (R.getNode()) {
160 assert(R.getNode() != N);
161 SetSoftenedFloat(SDValue(N, ResNo), R);
162 }
163}
164
165SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
166 bool IsStrict = N->isStrictFPOpcode();
167 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
168 unsigned Offset = IsStrict ? 1 : 0;
169 assert(N->getNumOperands() == (1 + Offset) &&
170 "Unexpected number of operands!");
171 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
172 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
174 EVT OpVT = N->getOperand(0 + Offset).getValueType();
175 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
176 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
177 CallOptions, SDLoc(N),
178 Chain);
179 if (IsStrict)
180 ReplaceValueWith(SDValue(N, 1), Tmp.second);
181 return Tmp.first;
182}
183
184SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
185 bool IsStrict = N->isStrictFPOpcode();
186 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
187 unsigned Offset = IsStrict ? 1 : 0;
188 assert(N->getNumOperands() == (2 + Offset) &&
189 "Unexpected number of operands!");
190 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
191 GetSoftenedFloat(N->getOperand(1 + Offset)) };
192 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
194 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
195 N->getOperand(1 + Offset).getValueType() };
196 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
197 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
198 CallOptions, SDLoc(N),
199 Chain);
200 if (IsStrict)
201 ReplaceValueWith(SDValue(N, 1), Tmp.second);
202 return Tmp.first;
203}
204
205SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
206 return BitConvertToInteger(N->getOperand(0));
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
210 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
211 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
212 GetSoftenedFloat(N->getOperand(0)));
213}
214
215SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
216 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
217 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
218 GetSoftenedFloat(N->getOperand(0)));
219 return NewFence;
220}
221
222SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
223 unsigned ResNo) {
224 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
225 return BitConvertToInteger(Op);
226}
227
228SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
229 // Convert the inputs to integers, and build a new pair out of them.
230 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
232 N->getValueType(0)),
233 BitConvertToInteger(N->getOperand(0)),
234 BitConvertToInteger(N->getOperand(1)));
235}
236
237SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
238 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
239 // In ppcf128, the high 64 bits are always first in memory regardless
240 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
241 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
242 // way. However, APInt's are serialized in an Endian-sensitive fashion,
243 // so on big-Endian targets, the two doubles are output in the wrong
244 // order. Fix this by manually flipping the order of the high 64 bits
245 // and the low 64 bits here.
246 if (DAG.getDataLayout().isBigEndian() &&
247 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
248 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
250 APInt Val(128, words);
251 return DAG.getConstant(Val, SDLoc(CN),
253 CN->getValueType(0)));
254 } else {
255 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
257 CN->getValueType(0)));
258 }
259}
260
261SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
262 SDValue Src = N->getOperand(0);
263 assert(Src.getValueType() == MVT::ppcf128 &&
264 "In floats only ppcf128 can be extracted by element!");
265 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
266 N->getValueType(0).changeTypeToInteger(),
267 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
268}
269
270SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
271 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
274 NewOp, N->getOperand(1));
275}
276
277SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
278 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
279 unsigned Size = NVT.getSizeInBits();
280
281 // Mask = ~(1 << (Size-1))
283 API.clearBit(Size - 1);
284 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
285 SDValue Op = GetSoftenedFloat(N->getOperand(0));
286 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
287}
288
289SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
290 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
291 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
292 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
293 RTLIB::FMIN_F32,
294 RTLIB::FMIN_F64,
295 RTLIB::FMIN_F80,
296 RTLIB::FMIN_F128,
297 RTLIB::FMIN_PPCF128));
298}
299
300SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
301 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
302 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
303 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
304 RTLIB::FMAX_F32,
305 RTLIB::FMAX_F64,
306 RTLIB::FMAX_F80,
307 RTLIB::FMAX_F128,
308 RTLIB::FMAX_PPCF128));
309}
310
311SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
312 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
313 RTLIB::ADD_F32,
314 RTLIB::ADD_F64,
315 RTLIB::ADD_F80,
316 RTLIB::ADD_F128,
317 RTLIB::ADD_PPCF128));
318}
319
320SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
321 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
322 RTLIB::CBRT_F32,
323 RTLIB::CBRT_F64,
324 RTLIB::CBRT_F80,
325 RTLIB::CBRT_F128,
326 RTLIB::CBRT_PPCF128));
327}
328
329SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
330 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
331 RTLIB::CEIL_F32,
332 RTLIB::CEIL_F64,
333 RTLIB::CEIL_F80,
334 RTLIB::CEIL_F128,
335 RTLIB::CEIL_PPCF128));
336}
337
338SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
339 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
340 SDValue RHS = BitConvertToInteger(N->getOperand(1));
341 SDLoc dl(N);
342
343 EVT LVT = LHS.getValueType();
344 EVT RVT = RHS.getValueType();
345
346 unsigned LSize = LVT.getSizeInBits();
347 unsigned RSize = RVT.getSizeInBits();
348
349 // First get the sign bit of second operand.
350 SDValue SignBit = DAG.getNode(
351 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
352 DAG.getConstant(RSize - 1, dl,
353 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
354 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
355
356 // Shift right or sign-extend it if the two operands have different types.
357 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
358 if (SizeDiff > 0) {
359 SignBit =
360 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
361 DAG.getConstant(SizeDiff, dl,
362 TLI.getShiftAmountTy(SignBit.getValueType(),
363 DAG.getDataLayout())));
364 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
365 } else if (SizeDiff < 0) {
366 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
367 SignBit =
368 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
369 DAG.getConstant(-SizeDiff, dl,
370 TLI.getShiftAmountTy(SignBit.getValueType(),
371 DAG.getDataLayout())));
372 }
373
374 // Clear the sign bit of the first operand.
375 SDValue Mask = DAG.getNode(
376 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
377 DAG.getConstant(LSize - 1, dl,
378 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
379 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
380 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
381
382 // Or the value with the sign bit.
383 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
384}
385
386SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
387 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
388 RTLIB::COS_F32,
389 RTLIB::COS_F64,
390 RTLIB::COS_F80,
391 RTLIB::COS_F128,
392 RTLIB::COS_PPCF128));
393}
394
395SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
396 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
397 RTLIB::DIV_F32,
398 RTLIB::DIV_F64,
399 RTLIB::DIV_F80,
400 RTLIB::DIV_F128,
401 RTLIB::DIV_PPCF128));
402}
403
404SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
405 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
406 RTLIB::EXP_F32,
407 RTLIB::EXP_F64,
408 RTLIB::EXP_F80,
409 RTLIB::EXP_F128,
410 RTLIB::EXP_PPCF128));
411}
412
413SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
414 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
415 RTLIB::EXP2_F32,
416 RTLIB::EXP2_F64,
417 RTLIB::EXP2_F80,
418 RTLIB::EXP2_F128,
419 RTLIB::EXP2_PPCF128));
420}
421
422SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
423 return SoftenFloatRes_Unary(
424 N,
425 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
426 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
427}
428
429SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
430 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
431 RTLIB::FLOOR_F32,
432 RTLIB::FLOOR_F64,
433 RTLIB::FLOOR_F80,
434 RTLIB::FLOOR_F128,
435 RTLIB::FLOOR_PPCF128));
436}
437
438SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
439 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
440 RTLIB::LOG_F32,
441 RTLIB::LOG_F64,
442 RTLIB::LOG_F80,
443 RTLIB::LOG_F128,
444 RTLIB::LOG_PPCF128));
445}
446
447SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
448 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
449 RTLIB::LOG2_F32,
450 RTLIB::LOG2_F64,
451 RTLIB::LOG2_F80,
452 RTLIB::LOG2_F128,
453 RTLIB::LOG2_PPCF128));
454}
455
456SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
457 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
458 RTLIB::LOG10_F32,
459 RTLIB::LOG10_F64,
460 RTLIB::LOG10_F80,
461 RTLIB::LOG10_F128,
462 RTLIB::LOG10_PPCF128));
463}
464
465SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
466 bool IsStrict = N->isStrictFPOpcode();
467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468 unsigned Offset = IsStrict ? 1 : 0;
469 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
470 GetSoftenedFloat(N->getOperand(1 + Offset)),
471 GetSoftenedFloat(N->getOperand(2 + Offset)) };
472 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
474 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
475 N->getOperand(1 + Offset).getValueType(),
476 N->getOperand(2 + Offset).getValueType() };
477 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
478 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
479 GetFPLibCall(N->getValueType(0),
480 RTLIB::FMA_F32,
481 RTLIB::FMA_F64,
482 RTLIB::FMA_F80,
483 RTLIB::FMA_F128,
484 RTLIB::FMA_PPCF128),
485 NVT, Ops, CallOptions, SDLoc(N), Chain);
486 if (IsStrict)
487 ReplaceValueWith(SDValue(N, 1), Tmp.second);
488 return Tmp.first;
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
492 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
493 RTLIB::MUL_F32,
494 RTLIB::MUL_F64,
495 RTLIB::MUL_F80,
496 RTLIB::MUL_F128,
497 RTLIB::MUL_PPCF128));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
501 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
502 RTLIB::NEARBYINT_F32,
503 RTLIB::NEARBYINT_F64,
504 RTLIB::NEARBYINT_F80,
505 RTLIB::NEARBYINT_F128,
506 RTLIB::NEARBYINT_PPCF128));
507}
508
509SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
511 SDLoc dl(N);
512
513 // Expand Y = FNEG(X) -> Y = X ^ sign mask
514 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
515 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
516 DAG.getConstant(SignMask, dl, NVT));
517}
518
519SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
520 bool IsStrict = N->isStrictFPOpcode();
521 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
522 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
523
524 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
525
526 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
527 Op = GetPromotedFloat(Op);
528 // If the promotion did the FP_EXTEND to the destination type for us,
529 // there's nothing left to do here.
530 if (Op.getValueType() == N->getValueType(0)) {
531 if (IsStrict)
532 ReplaceValueWith(SDValue(N, 1), Chain);
533 return BitConvertToInteger(Op);
534 }
535 }
536
537 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
538 // -> f32, so proceed in two stages. Also, it's entirely possible for both
539 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
540 // than FP16_TO_FP.
541 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
542 N->getValueType(0) != MVT::f32) {
543 if (IsStrict) {
545 { MVT::f32, MVT::Other }, { Chain, Op });
546 Chain = Op.getValue(1);
547 } else {
548 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
549 }
550 }
551
552 if (Op.getValueType() == MVT::bf16) {
553 // FIXME: Need ReplaceValueWith on chain in strict case
554 return SoftenFloatRes_BF16_TO_FP(N);
555 }
556
557 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
558 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
560 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
561 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
563 CallOptions, SDLoc(N),
564 Chain);
565 if (IsStrict)
566 ReplaceValueWith(SDValue(N, 1), Tmp.second);
567 return Tmp.first;
568}
569
570// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
571// nodes?
572SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
573 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
574 SDValue Op = N->getOperand(0);
576 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
577 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
578 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
579 CallOptions, SDLoc(N)).first;
580 if (N->getValueType(0) == MVT::f32)
581 return Res32;
582
583 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
585 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
586 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
587}
588
589// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
590// nodes?
591SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
592 assert(N->getValueType(0) == MVT::f32 &&
593 "Can only soften BF16_TO_FP with f32 result");
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
595 SDValue Op = N->getOperand(0);
596 SDLoc DL(N);
597 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
598 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
599 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
600 DAG.getShiftAmountConstant(16, NVT, DL));
601 return Res;
602}
603
604SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
605 bool IsStrict = N->isStrictFPOpcode();
606 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
607 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
608 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
609 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
610 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
612 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
613 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
614 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
615 CallOptions, SDLoc(N),
616 Chain);
617 if (IsStrict)
618 ReplaceValueWith(SDValue(N, 1), Tmp.second);
619 return Tmp.first;
620}
621
622SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
623 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
624 RTLIB::POW_F32,
625 RTLIB::POW_F64,
626 RTLIB::POW_F80,
627 RTLIB::POW_F128,
628 RTLIB::POW_PPCF128));
629}
630
631SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
632 bool IsStrict = N->isStrictFPOpcode();
633 unsigned Offset = IsStrict ? 1 : 0;
634 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
635 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
636 "Unsupported power type!");
637 bool IsPowI =
638 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
639
640 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
641 : RTLIB::getLDEXP(N->getValueType(0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
643 if (!TLI.getLibcallName(LC)) {
644 // Some targets don't have a powi libcall; use pow instead.
645 // FIXME: Implement this if some target needs it.
646 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
647 return DAG.getUNDEF(N->getValueType(0));
648 }
649
650 if (DAG.getLibInfo().getIntSize() !=
651 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
652 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
653 // would use the wrong type for the argument.
654 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
655 return DAG.getUNDEF(N->getValueType(0));
656 }
657
658 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
659 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
660 N->getOperand(1 + Offset) };
661 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
663 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
664 N->getOperand(1 + Offset).getValueType() };
665 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
666 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
667 CallOptions, SDLoc(N),
668 Chain);
669 if (IsStrict)
670 ReplaceValueWith(SDValue(N, 1), Tmp.second);
671 return Tmp.first;
672}
673
674SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
675 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
676 EVT VT0 = N->getValueType(0);
677 EVT VT1 = N->getValueType(1);
679
680 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
681 // If the exponent does not match with sizeof(int) a libcall would use the
682 // wrong type for the argument.
683 // TODO: Should be able to handle mismatches.
684 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
685 return DAG.getUNDEF(N->getValueType(0));
686 }
687
688 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
689 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
690
691 SDLoc DL(N);
692
694 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
695 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
696
697 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
698 // but we only really need to handle the 0th one for softening anyway.
699 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
700
701 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
702 /*Chain=*/SDValue());
703 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
704 auto PtrInfo =
706
707 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
708
709 ReplaceValueWith(SDValue(N, 1), LoadExp);
710 return ReturnVal;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
714 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
715 RTLIB::REM_F32,
716 RTLIB::REM_F64,
717 RTLIB::REM_F80,
718 RTLIB::REM_F128,
719 RTLIB::REM_PPCF128));
720}
721
722SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
723 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
724 RTLIB::RINT_F32,
725 RTLIB::RINT_F64,
726 RTLIB::RINT_F80,
727 RTLIB::RINT_F128,
728 RTLIB::RINT_PPCF128));
729}
730
731SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
732 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
733 RTLIB::ROUND_F32,
734 RTLIB::ROUND_F64,
735 RTLIB::ROUND_F80,
736 RTLIB::ROUND_F128,
737 RTLIB::ROUND_PPCF128));
738}
739
740SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
741 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
742 RTLIB::ROUNDEVEN_F32,
743 RTLIB::ROUNDEVEN_F64,
744 RTLIB::ROUNDEVEN_F80,
745 RTLIB::ROUNDEVEN_F128,
746 RTLIB::ROUNDEVEN_PPCF128));
747}
748
749SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
750 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
751 RTLIB::SIN_F32,
752 RTLIB::SIN_F64,
753 RTLIB::SIN_F80,
754 RTLIB::SIN_F128,
755 RTLIB::SIN_PPCF128));
756}
757
758SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
759 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
760 RTLIB::SQRT_F32,
761 RTLIB::SQRT_F64,
762 RTLIB::SQRT_F80,
763 RTLIB::SQRT_F128,
764 RTLIB::SQRT_PPCF128));
765}
766
767SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
768 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
769 RTLIB::SUB_F32,
770 RTLIB::SUB_F64,
771 RTLIB::SUB_F80,
772 RTLIB::SUB_F128,
773 RTLIB::SUB_PPCF128));
774}
775
776SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
777 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
778 RTLIB::TRUNC_F32,
779 RTLIB::TRUNC_F64,
780 RTLIB::TRUNC_F80,
781 RTLIB::TRUNC_F128,
782 RTLIB::TRUNC_PPCF128));
783}
784
785SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
786 LoadSDNode *L = cast<LoadSDNode>(N);
787 EVT VT = N->getValueType(0);
788 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
789 SDLoc dl(N);
790
791 auto MMOFlags =
792 L->getMemOperand()->getFlags() &
794 SDValue NewL;
795 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
796 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
797 L->getChain(), L->getBasePtr(), L->getOffset(),
798 L->getPointerInfo(), NVT, L->getOriginalAlign(),
799 MMOFlags, L->getAAInfo());
800 // Legalized the chain result - switch anything that used the old chain to
801 // use the new one.
802 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
803 return NewL;
804 }
805
806 // Do a non-extending load followed by FP_EXTEND.
807 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
808 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
809 L->getPointerInfo(), L->getMemoryVT(),
810 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
811 // Legalized the chain result - switch anything that used the old chain to
812 // use the new one.
813 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
814 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
815 return BitConvertToInteger(ExtendNode);
816}
817
818SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
819 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
820 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
821 return DAG.getSelect(SDLoc(N),
822 LHS.getValueType(), N->getOperand(0), LHS, RHS);
823}
824
825SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
826 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
827 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
828 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
829 LHS.getValueType(), N->getOperand(0),
830 N->getOperand(1), LHS, RHS, N->getOperand(4));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
834 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
835 N->getValueType(0)));
836}
837
838SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
839 SDValue Chain = N->getOperand(0); // Get the chain.
840 SDValue Ptr = N->getOperand(1); // Get the pointer.
841 EVT VT = N->getValueType(0);
842 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
843 SDLoc dl(N);
844
845 SDValue NewVAARG;
846 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
847 N->getConstantOperandVal(3));
848
849 // Legalized the chain result - switch anything that used the old chain to
850 // use the new one.
851 if (N != NewVAARG.getValue(1).getNode())
852 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
853 return NewVAARG;
854}
855
856SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
857 bool IsStrict = N->isStrictFPOpcode();
858 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
859 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
860 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
861 EVT RVT = N->getValueType(0);
862 EVT NVT = EVT();
863 SDLoc dl(N);
864
865 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
866 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
867 // match. Look for an appropriate libcall.
868 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
869 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
870 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
871 NVT = (MVT::SimpleValueType)t;
872 // The source needs to big enough to hold the operand.
873 if (NVT.bitsGE(SVT))
874 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
875 }
876 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
877
878 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
879 // Sign/zero extend the argument if the libcall takes a larger type.
881 NVT, N->getOperand(IsStrict ? 1 : 0));
883 CallOptions.setSExt(Signed);
884 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
885 std::pair<SDValue, SDValue> Tmp =
886 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
887 Op, CallOptions, dl, Chain);
888
889 if (IsStrict)
890 ReplaceValueWith(SDValue(N, 1), Tmp.second);
891 return Tmp.first;
892}
893
894SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
895 // Expand and soften recursively.
896 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
897 return SDValue();
898}
899
900SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
901 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
902 return SDValue();
903}
904
905//===----------------------------------------------------------------------===//
906// Convert Float Operand to Integer
907//===----------------------------------------------------------------------===//
908
909bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
910 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
911 SDValue Res = SDValue();
912
913 switch (N->getOpcode()) {
914 default:
915#ifndef NDEBUG
916 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
917 N->dump(&DAG); dbgs() << "\n";
918#endif
919 report_fatal_error("Do not know how to soften this operator's operand!");
920
921 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
922 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
924 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
925 case ISD::FP_TO_BF16:
928 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
931 case ISD::FP_TO_SINT:
932 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
935 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
937 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
939 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
941 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
943 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
944 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
947 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
948 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
949 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
950 }
951
952 // If the result is null, the sub-method took care of registering results etc.
953 if (!Res.getNode()) return false;
954
955 // If the result is N, the sub-method updated N in place. Tell the legalizer
956 // core about this to re-analyze.
957 if (Res.getNode() == N)
958 return true;
959
960 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
961 "Invalid operand softening");
962
963 ReplaceValueWith(SDValue(N, 0), Res);
964 return false;
965}
966
967SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
968 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
969
970 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
971}
972
973SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
974 // We actually deal with the partially-softened FP_TO_FP16 node too, which
975 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
976 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
977 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
978 N->getOpcode() == ISD::FP_TO_BF16 ||
979 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
980 N->getOpcode() == ISD::STRICT_FP_ROUND);
981
982 bool IsStrict = N->isStrictFPOpcode();
983 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
984 EVT SVT = Op.getValueType();
985 EVT RVT = N->getValueType(0);
986 EVT FloatRVT = RVT;
987 if (N->getOpcode() == ISD::FP_TO_FP16 ||
988 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
989 FloatRVT = MVT::f16;
990 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
991 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
992 FloatRVT = MVT::bf16;
993
994 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
995 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
996
997 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
998 Op = GetSoftenedFloat(Op);
1000 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1001 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1002 CallOptions, SDLoc(N),
1003 Chain);
1004 if (IsStrict) {
1005 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1006 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1007 return SDValue();
1008 }
1009 return Tmp.first;
1010}
1011
1012SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1013 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1014 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1015
1016 EVT VT = NewLHS.getValueType();
1017 NewLHS = GetSoftenedFloat(NewLHS);
1018 NewRHS = GetSoftenedFloat(NewRHS);
1019 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1020 N->getOperand(2), N->getOperand(3));
1021
1022 // If softenSetCCOperands returned a scalar, we need to compare the result
1023 // against zero to select between true and false values.
1024 if (!NewRHS.getNode()) {
1025 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1026 CCCode = ISD::SETNE;
1027 }
1028
1029 // Update N to have the operands specified.
1030 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1031 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1032 N->getOperand(4)),
1033 0);
1034}
1035
1036// Even if the result type is legal, no libcall may exactly match. (e.g. We
1037// don't have FP-i8 conversions) This helper method looks for an appropriate
1038// promoted libcall.
1039static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1040 bool Signed) {
1041 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1042 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1043 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1044 ++IntVT) {
1045 Promoted = (MVT::SimpleValueType)IntVT;
1046 // The type needs to big enough to hold the result.
1047 if (Promoted.bitsGE(RetVT))
1048 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1049 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1050 }
1051 return LC;
1052}
1053
1054SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1055 bool IsStrict = N->isStrictFPOpcode();
1056 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1058
1059 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1060 EVT SVT = Op.getValueType();
1061 EVT RVT = N->getValueType(0);
1062 EVT NVT = EVT();
1063 SDLoc dl(N);
1064
1065 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1066 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1067 // match, eg. we don't have fp -> i8 conversions.
1068 // Look for an appropriate libcall.
1069 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1070 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1071 "Unsupported FP_TO_XINT!");
1072
1073 Op = GetSoftenedFloat(Op);
1074 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1076 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1077 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1078 CallOptions, dl, Chain);
1079
1080 // Truncate the result if the libcall returns a larger type.
1081 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1082
1083 if (!IsStrict)
1084 return Res;
1085
1086 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1087 ReplaceValueWith(SDValue(N, 0), Res);
1088 return SDValue();
1089}
1090
1091SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1092 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1093 return Res;
1094}
1095
1096SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1097 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1098 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1099
1100 EVT VT = NewLHS.getValueType();
1101 NewLHS = GetSoftenedFloat(NewLHS);
1102 NewRHS = GetSoftenedFloat(NewRHS);
1103 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1104 N->getOperand(0), N->getOperand(1));
1105
1106 // If softenSetCCOperands returned a scalar, we need to compare the result
1107 // against zero to select between true and false values.
1108 if (!NewRHS.getNode()) {
1109 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1110 CCCode = ISD::SETNE;
1111 }
1112
1113 // Update N to have the operands specified.
1114 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1115 N->getOperand(2), N->getOperand(3),
1116 DAG.getCondCode(CCCode)),
1117 0);
1118}
1119
1120SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1121 bool IsStrict = N->isStrictFPOpcode();
1122 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1123 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1124 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1125 ISD::CondCode CCCode =
1126 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1127
1128 EVT VT = Op0.getValueType();
1129 SDValue NewLHS = GetSoftenedFloat(Op0);
1130 SDValue NewRHS = GetSoftenedFloat(Op1);
1131 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1132 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1133
1134 // Update N to have the operands specified.
1135 if (NewRHS.getNode()) {
1136 if (IsStrict)
1137 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1138 NewRHS, DAG.getCondCode(CCCode));
1139 else
1140 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1141 DAG.getCondCode(CCCode)), 0);
1142 }
1143
1144 // Otherwise, softenSetCCOperands returned a scalar, use it.
1145 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1146 "Unexpected setcc expansion!");
1147
1148 if (IsStrict) {
1149 ReplaceValueWith(SDValue(N, 0), NewLHS);
1150 ReplaceValueWith(SDValue(N, 1), Chain);
1151 return SDValue();
1152 }
1153 return NewLHS;
1154}
1155
1156SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1157 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1158 assert(OpNo == 1 && "Can only soften the stored value!");
1159 StoreSDNode *ST = cast<StoreSDNode>(N);
1160 SDValue Val = ST->getValue();
1161 SDLoc dl(N);
1162
1163 if (ST->isTruncatingStore())
1164 // Do an FP_ROUND followed by a non-truncating store.
1165 Val = BitConvertToInteger(
1166 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1167 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1168 else
1169 Val = GetSoftenedFloat(Val);
1170
1171 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1172 ST->getMemOperand());
1173}
1174
1175SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1176 SDValue LHS = N->getOperand(0);
1177 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1178 SDLoc dl(N);
1179
1180 EVT LVT = LHS.getValueType();
1181 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1182 EVT RVT = RHS.getValueType();
1183
1184 unsigned LSize = LVT.getSizeInBits();
1185 unsigned RSize = RVT.getSizeInBits();
1186
1187 // Shift right or sign-extend it if the two operands have different types.
1188 int SizeDiff = RSize - LSize;
1189 if (SizeDiff > 0) {
1190 RHS =
1191 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1192 DAG.getConstant(SizeDiff, dl,
1193 TLI.getShiftAmountTy(RHS.getValueType(),
1194 DAG.getDataLayout())));
1195 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1196 } else if (SizeDiff < 0) {
1197 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1198 RHS =
1199 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1200 DAG.getConstant(-SizeDiff, dl,
1201 TLI.getShiftAmountTy(RHS.getValueType(),
1202 DAG.getDataLayout())));
1203 }
1204
1205 RHS = DAG.getBitcast(LVT, RHS);
1206 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1207}
1208
1209SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1210 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1211 bool IsStrict = N->isStrictFPOpcode();
1212 unsigned Offset = IsStrict ? 1 : 0;
1213 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1214 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1216 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1217 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1218 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1219 CallOptions, SDLoc(N),
1220 Chain);
1221 if (IsStrict) {
1222 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1223 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1224 return SDValue();
1225 }
1226
1227 return Tmp.first;
1228}
1229
1230SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1231 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1232 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1233 RTLIB::LROUND_F32,
1234 RTLIB::LROUND_F64,
1235 RTLIB::LROUND_F80,
1236 RTLIB::LROUND_F128,
1237 RTLIB::LROUND_PPCF128));
1238}
1239
1240SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1241 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1242 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1243 RTLIB::LLROUND_F32,
1244 RTLIB::LLROUND_F64,
1245 RTLIB::LLROUND_F80,
1246 RTLIB::LLROUND_F128,
1247 RTLIB::LLROUND_PPCF128));
1248}
1249
1250SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1251 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1252 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1253 RTLIB::LRINT_F32,
1254 RTLIB::LRINT_F64,
1255 RTLIB::LRINT_F80,
1256 RTLIB::LRINT_F128,
1257 RTLIB::LRINT_PPCF128));
1258}
1259
1260SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1261 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1262 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1263 RTLIB::LLRINT_F32,
1264 RTLIB::LLRINT_F64,
1265 RTLIB::LLRINT_F80,
1266 RTLIB::LLRINT_F128,
1267 RTLIB::LLRINT_PPCF128));
1268}
1269
1270//===----------------------------------------------------------------------===//
1271// Float Result Expansion
1272//===----------------------------------------------------------------------===//
1273
1274/// ExpandFloatResult - This method is called when the specified result of the
1275/// specified node is found to need expansion. At this point, the node may also
1276/// have invalid operands or may have other results that need promotion, we just
1277/// know that (at least) one result needs expansion.
1278void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1279 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1280 SDValue Lo, Hi;
1281 Lo = Hi = SDValue();
1282
1283 // See if the target wants to custom expand this node.
1284 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1285 return;
1286
1287 switch (N->getOpcode()) {
1288 default:
1289#ifndef NDEBUG
1290 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1291 N->dump(&DAG); dbgs() << "\n";
1292#endif
1293 report_fatal_error("Do not know how to expand the result of this "
1294 "operator!");
1295
1296 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1297 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1298 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1299
1300 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1301 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1302 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1303 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1304 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1305 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1306
1307 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1308 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1310 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1312 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1313 case ISD::STRICT_FADD:
1314 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1315 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1316 case ISD::STRICT_FCEIL:
1317 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1318 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1319 case ISD::STRICT_FCOS:
1320 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1321 case ISD::STRICT_FDIV:
1322 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1323 case ISD::STRICT_FEXP:
1324 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1325 case ISD::STRICT_FEXP2:
1326 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1327 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1328 case ISD::STRICT_FFLOOR:
1329 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1330 case ISD::STRICT_FLOG:
1331 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1332 case ISD::STRICT_FLOG2:
1333 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1334 case ISD::STRICT_FLOG10:
1335 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1336 case ISD::STRICT_FMA:
1337 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1338 case ISD::STRICT_FMUL:
1339 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1341 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1342 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1344 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1345 case ISD::STRICT_FPOW:
1346 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1347 case ISD::STRICT_FPOWI:
1348 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1349 case ISD::FLDEXP:
1350 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1351 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1352 case ISD::STRICT_FRINT:
1353 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1354 case ISD::STRICT_FROUND:
1355 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1357 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1358 case ISD::STRICT_FSIN:
1359 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1360 case ISD::STRICT_FSQRT:
1361 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1362 case ISD::STRICT_FSUB:
1363 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1364 case ISD::STRICT_FTRUNC:
1365 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1366 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1369 case ISD::SINT_TO_FP:
1370 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1371 case ISD::STRICT_FREM:
1372 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1373 }
1374
1375 // If Lo/Hi is null, the sub-method took care of registering results etc.
1376 if (Lo.getNode())
1377 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1378}
1379
1380void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1381 SDValue &Hi) {
1382 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1383 assert(NVT.getSizeInBits() == 64 &&
1384 "Do not know how to expand this float constant!");
1385 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1386 SDLoc dl(N);
1388 APInt(64, C.getRawData()[1])),
1389 dl, NVT);
1391 APInt(64, C.getRawData()[0])),
1392 dl, NVT);
1393}
1394
1395void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1396 SDValue &Lo, SDValue &Hi) {
1397 bool IsStrict = N->isStrictFPOpcode();
1398 unsigned Offset = IsStrict ? 1 : 0;
1399 SDValue Op = N->getOperand(0 + Offset);
1400 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1402 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1403 Op, CallOptions, SDLoc(N),
1404 Chain);
1405 if (IsStrict)
1406 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1407 GetPairElements(Tmp.first, Lo, Hi);
1408}
1409
1410void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1411 SDValue &Lo, SDValue &Hi) {
1412 bool IsStrict = N->isStrictFPOpcode();
1413 unsigned Offset = IsStrict ? 1 : 0;
1414 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1415 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1417 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1418 Ops, CallOptions, SDLoc(N),
1419 Chain);
1420 if (IsStrict)
1421 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1422 GetPairElements(Tmp.first, Lo, Hi);
1423}
1424
1425void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1426 SDValue &Hi) {
1427 assert(N->getValueType(0) == MVT::ppcf128 &&
1428 "Logic only correct for ppcf128!");
1429 SDLoc dl(N);
1430 SDValue Tmp;
1431 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1432 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1433 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1434 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1435 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1436 ISD::SETEQ);
1437}
1438
1439void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1440 SDValue &Hi) {
1441 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1442 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1443 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1444 RTLIB::FMIN_PPCF128), Lo, Hi);
1445}
1446
1447void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1448 SDValue &Hi) {
1449 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1450 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1451 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1452 RTLIB::FMAX_PPCF128), Lo, Hi);
1453}
1454
1455void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1456 SDValue &Hi) {
1457 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1458 RTLIB::ADD_F32, RTLIB::ADD_F64,
1459 RTLIB::ADD_F80, RTLIB::ADD_F128,
1460 RTLIB::ADD_PPCF128), Lo, Hi);
1461}
1462
1463void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1464 SDValue &Hi) {
1465 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1466 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1467 RTLIB::CBRT_F128,
1468 RTLIB::CBRT_PPCF128), Lo, Hi);
1469}
1470
1471void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1472 SDValue &Lo, SDValue &Hi) {
1473 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1474 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1475 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1476 RTLIB::CEIL_PPCF128), Lo, Hi);
1477}
1478
1479void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1480 SDValue &Lo, SDValue &Hi) {
1481 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1482 RTLIB::COPYSIGN_F32,
1483 RTLIB::COPYSIGN_F64,
1484 RTLIB::COPYSIGN_F80,
1485 RTLIB::COPYSIGN_F128,
1486 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1487}
1488
1489void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1490 SDValue &Lo, SDValue &Hi) {
1491 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1492 RTLIB::COS_F32, RTLIB::COS_F64,
1493 RTLIB::COS_F80, RTLIB::COS_F128,
1494 RTLIB::COS_PPCF128), Lo, Hi);
1495}
1496
1497void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1498 SDValue &Hi) {
1499 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1500 RTLIB::DIV_F32,
1501 RTLIB::DIV_F64,
1502 RTLIB::DIV_F80,
1503 RTLIB::DIV_F128,
1504 RTLIB::DIV_PPCF128), Lo, Hi);
1505}
1506
1507void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1508 SDValue &Lo, SDValue &Hi) {
1509 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1510 RTLIB::EXP_F32, RTLIB::EXP_F64,
1511 RTLIB::EXP_F80, RTLIB::EXP_F128,
1512 RTLIB::EXP_PPCF128), Lo, Hi);
1513}
1514
1515void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1516 SDValue &Lo, SDValue &Hi) {
1517 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1518 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1519 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1520 RTLIB::EXP2_PPCF128), Lo, Hi);
1521}
1522
1523void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1524 SDValue &Hi) {
1525 ExpandFloatRes_Unary(N,
1526 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1527 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1528 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1529 Lo, Hi);
1530}
1531
1532void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1533 SDValue &Lo, SDValue &Hi) {
1534 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1535 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1536 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1537 RTLIB::FLOOR_PPCF128), Lo, Hi);
1538}
1539
1540void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1541 SDValue &Lo, SDValue &Hi) {
1542 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1543 RTLIB::LOG_F32, RTLIB::LOG_F64,
1544 RTLIB::LOG_F80, RTLIB::LOG_F128,
1545 RTLIB::LOG_PPCF128), Lo, Hi);
1546}
1547
1548void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1549 SDValue &Lo, SDValue &Hi) {
1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1551 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1552 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1553 RTLIB::LOG2_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1560 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1561 RTLIB::LOG10_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 bool IsStrict = N->isStrictFPOpcode();
1567 unsigned Offset = IsStrict ? 1 : 0;
1568 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1569 N->getOperand(2 + Offset) };
1570 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1572 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1573 RTLIB::FMA_F32,
1574 RTLIB::FMA_F64,
1575 RTLIB::FMA_F80,
1576 RTLIB::FMA_F128,
1577 RTLIB::FMA_PPCF128),
1578 N->getValueType(0), Ops, CallOptions,
1579 SDLoc(N), Chain);
1580 if (IsStrict)
1581 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1582 GetPairElements(Tmp.first, Lo, Hi);
1583}
1584
1585void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1586 SDValue &Hi) {
1587 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1588 RTLIB::MUL_F32,
1589 RTLIB::MUL_F64,
1590 RTLIB::MUL_F80,
1591 RTLIB::MUL_F128,
1592 RTLIB::MUL_PPCF128), Lo, Hi);
1593}
1594
1595void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1596 SDValue &Lo, SDValue &Hi) {
1597 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1598 RTLIB::NEARBYINT_F32,
1599 RTLIB::NEARBYINT_F64,
1600 RTLIB::NEARBYINT_F80,
1601 RTLIB::NEARBYINT_F128,
1602 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1603}
1604
1605void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1606 SDValue &Hi) {
1607 SDLoc dl(N);
1608 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1609 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1610 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1611}
1612
1613void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1614 SDValue &Hi) {
1615 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1616 SDLoc dl(N);
1617 bool IsStrict = N->isStrictFPOpcode();
1618
1619 SDValue Chain;
1620 if (IsStrict) {
1621 // If the expanded type is the same as the input type, just bypass the node.
1622 if (NVT == N->getOperand(1).getValueType()) {
1623 Hi = N->getOperand(1);
1624 Chain = N->getOperand(0);
1625 } else {
1626 // Other we need to extend.
1627 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1628 { N->getOperand(0), N->getOperand(1) });
1629 Chain = Hi.getValue(1);
1630 }
1631 } else {
1632 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1633 }
1634
1636 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1637
1638 if (IsStrict)
1639 ReplaceValueWith(SDValue(N, 1), Chain);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1643 SDValue &Lo, SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1645 RTLIB::POW_F32, RTLIB::POW_F64,
1646 RTLIB::POW_F80, RTLIB::POW_F128,
1647 RTLIB::POW_PPCF128), Lo, Hi);
1648}
1649
1650void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1651 SDValue &Lo, SDValue &Hi) {
1652 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1653}
1654
1655void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1656 SDValue &Hi) {
1657 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1658}
1659
1660void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 assert(N->getValueType(0) == MVT::ppcf128 &&
1663 "Logic only correct for ppcf128!");
1664
1665 SDLoc dl(N);
1666 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1667 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1668 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1669}
1670
1671void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1672 SDValue &Lo, SDValue &Hi) {
1673 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1674 RTLIB::REM_F32, RTLIB::REM_F64,
1675 RTLIB::REM_F80, RTLIB::REM_F128,
1676 RTLIB::REM_PPCF128), Lo, Hi);
1677}
1678
1679void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1680 SDValue &Lo, SDValue &Hi) {
1681 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1682 RTLIB::RINT_F32, RTLIB::RINT_F64,
1683 RTLIB::RINT_F80, RTLIB::RINT_F128,
1684 RTLIB::RINT_PPCF128), Lo, Hi);
1685}
1686
1687void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1688 SDValue &Lo, SDValue &Hi) {
1689 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1690 RTLIB::ROUND_F32,
1691 RTLIB::ROUND_F64,
1692 RTLIB::ROUND_F80,
1693 RTLIB::ROUND_F128,
1694 RTLIB::ROUND_PPCF128), Lo, Hi);
1695}
1696
1697void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1698 SDValue &Lo, SDValue &Hi) {
1699 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1700 RTLIB::ROUNDEVEN_F32,
1701 RTLIB::ROUNDEVEN_F64,
1702 RTLIB::ROUNDEVEN_F80,
1703 RTLIB::ROUNDEVEN_F128,
1704 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1705}
1706
1707void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1708 SDValue &Lo, SDValue &Hi) {
1709 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1710 RTLIB::SIN_F32, RTLIB::SIN_F64,
1711 RTLIB::SIN_F80, RTLIB::SIN_F128,
1712 RTLIB::SIN_PPCF128), Lo, Hi);
1713}
1714
1715void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1716 SDValue &Lo, SDValue &Hi) {
1717 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1718 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1719 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1720 RTLIB::SQRT_PPCF128), Lo, Hi);
1721}
1722
1723void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1724 SDValue &Hi) {
1725 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1726 RTLIB::SUB_F32,
1727 RTLIB::SUB_F64,
1728 RTLIB::SUB_F80,
1729 RTLIB::SUB_F128,
1730 RTLIB::SUB_PPCF128), Lo, Hi);
1731}
1732
1733void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1734 SDValue &Lo, SDValue &Hi) {
1735 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1736 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1737 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1738 RTLIB::TRUNC_PPCF128), Lo, Hi);
1739}
1740
1741void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1743 if (ISD::isNormalLoad(N)) {
1744 ExpandRes_NormalLoad(N, Lo, Hi);
1745 return;
1746 }
1747
1748 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1749 LoadSDNode *LD = cast<LoadSDNode>(N);
1750 SDValue Chain = LD->getChain();
1751 SDValue Ptr = LD->getBasePtr();
1752 SDLoc dl(N);
1753
1754 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1755 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1756 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1757
1758 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1759 LD->getMemoryVT(), LD->getMemOperand());
1760
1761 // Remember the chain.
1762 Chain = Hi.getValue(1);
1763
1764 // The low part is zero.
1766 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1767
1768 // Modified the chain - switch anything that used the old chain to use the
1769 // new one.
1770 ReplaceValueWith(SDValue(LD, 1), Chain);
1771}
1772
1773void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1774 SDValue &Hi) {
1775 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1776 EVT VT = N->getValueType(0);
1777 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1778 bool Strict = N->isStrictFPOpcode();
1779 SDValue Src = N->getOperand(Strict ? 1 : 0);
1780 EVT SrcVT = Src.getValueType();
1781 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1782 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1783 SDLoc dl(N);
1784 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1785
1786 // TODO: Any other flags to propagate?
1788 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1789
1790 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1791 // When promoting partial word types to i32 we must honor the signedness,
1792 // though.
1793 if (SrcVT.bitsLE(MVT::i32)) {
1794 // The integer can be represented exactly in an f64.
1796 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1797 if (Strict) {
1798 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1799 {Chain, Src}, Flags);
1800 Chain = Hi.getValue(1);
1801 } else
1802 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1803 } else {
1804 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1805 if (SrcVT.bitsLE(MVT::i64)) {
1807 MVT::i64, Src);
1808 LC = RTLIB::SINTTOFP_I64_PPCF128;
1809 } else if (SrcVT.bitsLE(MVT::i128)) {
1810 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1811 LC = RTLIB::SINTTOFP_I128_PPCF128;
1812 }
1813 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1814
1816 CallOptions.setSExt(true);
1817 std::pair<SDValue, SDValue> Tmp =
1818 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1819 if (Strict)
1820 Chain = Tmp.second;
1821 GetPairElements(Tmp.first, Lo, Hi);
1822 }
1823
1824 // No need to complement for unsigned 32-bit integers
1825 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1826 if (Strict)
1827 ReplaceValueWith(SDValue(N, 1), Chain);
1828
1829 return;
1830 }
1831
1832 // Unsigned - fix up the SINT_TO_FP value just calculated.
1833 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1834 // keep semantics correctness if the integer is not exactly representable
1835 // here. See ExpandLegalINT_TO_FP.
1836 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1837 SrcVT = Src.getValueType();
1838
1839 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1840 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1841 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1842 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1843 ArrayRef<uint64_t> Parts;
1844
1845 switch (SrcVT.getSimpleVT().SimpleTy) {
1846 default:
1847 llvm_unreachable("Unsupported UINT_TO_FP!");
1848 case MVT::i32:
1849 Parts = TwoE32;
1850 break;
1851 case MVT::i64:
1852 Parts = TwoE64;
1853 break;
1854 case MVT::i128:
1855 Parts = TwoE128;
1856 break;
1857 }
1858
1859 // TODO: Are there other fast-math-flags to propagate to this FADD?
1860 SDValue NewLo = DAG.getConstantFP(
1861 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1862 if (Strict) {
1863 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1864 {Chain, Hi, NewLo}, Flags);
1865 Chain = Lo.getValue(1);
1866 ReplaceValueWith(SDValue(N, 1), Chain);
1867 } else
1868 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1869 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1870 Lo, Hi, ISD::SETLT);
1871 GetPairElements(Lo, Lo, Hi);
1872}
1873
1874
1875//===----------------------------------------------------------------------===//
1876// Float Operand Expansion
1877//===----------------------------------------------------------------------===//
1878
1879/// ExpandFloatOperand - This method is called when the specified operand of the
1880/// specified node is found to need expansion. At this point, all of the result
1881/// types of the node are known to be legal, but other operands of the node may
1882/// need promotion or expansion as well as the specified one.
1883bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1884 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1885 SDValue Res = SDValue();
1886
1887 // See if the target wants to custom expand this node.
1888 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1889 return false;
1890
1891 switch (N->getOpcode()) {
1892 default:
1893#ifndef NDEBUG
1894 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1895 N->dump(&DAG); dbgs() << "\n";
1896#endif
1897 report_fatal_error("Do not know how to expand this operator's operand!");
1898
1899 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1900 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1901 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1902
1903 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1904 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1906 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1909 case ISD::FP_TO_SINT:
1910 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1911 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1912 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1913 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1914 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1915 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1916 case ISD::STRICT_FSETCC:
1918 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1919 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1920 OpNo); break;
1921 }
1922
1923 // If the result is null, the sub-method took care of registering results etc.
1924 if (!Res.getNode()) return false;
1925
1926 // If the result is N, the sub-method updated N in place. Tell the legalizer
1927 // core about this.
1928 if (Res.getNode() == N)
1929 return true;
1930
1931 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1932 "Invalid operand expansion");
1933
1934 ReplaceValueWith(SDValue(N, 0), Res);
1935 return false;
1936}
1937
1938/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1939/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1940void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1941 SDValue &NewRHS,
1942 ISD::CondCode &CCCode,
1943 const SDLoc &dl, SDValue &Chain,
1944 bool IsSignaling) {
1945 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1946 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1947 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1948
1949 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1950
1951 // FIXME: This generated code sucks. We want to generate
1952 // FCMPU crN, hi1, hi2
1953 // BNE crN, L:
1954 // FCMPU crN, lo1, lo2
1955 // The following can be improved, but not that much.
1956 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1957 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1958 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1959 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1960 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1961 RHSLo, CCCode, OutputChain, IsSignaling);
1962 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1963 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1964 Tmp1 =
1965 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1966 ISD::SETUNE, OutputChain, IsSignaling);
1967 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1968 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1969 RHSHi, CCCode, OutputChain, IsSignaling);
1970 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1971 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1972 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1973 NewRHS = SDValue(); // LHS is the result, not a compare.
1974 Chain = OutputChain;
1975}
1976
1977SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1978 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1979 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1980 SDValue Chain;
1981 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1982
1983 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1984 // against zero to select between true and false values.
1985 if (!NewRHS.getNode()) {
1986 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1987 CCCode = ISD::SETNE;
1988 }
1989
1990 // Update N to have the operands specified.
1991 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1992 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1993 N->getOperand(4)), 0);
1994}
1995
1996SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1997 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1998 "Logic only correct for ppcf128!");
1999 SDValue Lo, Hi;
2000 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2001 // The ppcf128 value is providing only the sign; take it from the
2002 // higher-order double (which must have the larger magnitude).
2003 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2004 N->getValueType(0), N->getOperand(0), Hi);
2005}
2006
2007SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2008 bool IsStrict = N->isStrictFPOpcode();
2009 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2010 "Logic only correct for ppcf128!");
2011 SDValue Lo, Hi;
2012 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2013
2014 if (!IsStrict)
2015 // Round it the rest of the way (e.g. to f32) if needed.
2016 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2017 N->getValueType(0), Hi, N->getOperand(1));
2018
2019 // Eliminate the node if the input float type is the same as the output float
2020 // type.
2021 if (Hi.getValueType() == N->getValueType(0)) {
2022 // Connect the output chain to the input chain, unlinking the node.
2023 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2024 ReplaceValueWith(SDValue(N, 0), Hi);
2025 return SDValue();
2026 }
2027
2029 {N->getValueType(0), MVT::Other},
2030 {N->getOperand(0), Hi, N->getOperand(2)});
2031 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2032 ReplaceValueWith(SDValue(N, 0), Expansion);
2033 return SDValue();
2034}
2035
2036SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2037 EVT RVT = N->getValueType(0);
2038 SDLoc dl(N);
2039
2040 bool IsStrict = N->isStrictFPOpcode();
2041 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2042 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2043 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2044 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2045
2046 EVT NVT;
2047 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2048 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2049 "Unsupported FP_TO_XINT!");
2051 std::pair<SDValue, SDValue> Tmp =
2052 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2053 if (!IsStrict)
2054 return Tmp.first;
2055
2056 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2057 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2058 return SDValue();
2059}
2060
2061SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2062 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2063 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2064 SDValue Chain;
2065 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2066
2067 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2068 // against zero to select between true and false values.
2069 if (!NewRHS.getNode()) {
2070 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2071 CCCode = ISD::SETNE;
2072 }
2073
2074 // Update N to have the operands specified.
2075 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2076 N->getOperand(2), N->getOperand(3),
2077 DAG.getCondCode(CCCode)), 0);
2078}
2079
2080SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2081 bool IsStrict = N->isStrictFPOpcode();
2082 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2083 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2084 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2085 ISD::CondCode CCCode =
2086 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2087 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2088 N->getOpcode() == ISD::STRICT_FSETCCS);
2089
2090 // FloatExpandSetCCOperands always returned a scalar.
2091 assert(!NewRHS.getNode() && "Expect to return scalar");
2092 assert(NewLHS.getValueType() == N->getValueType(0) &&
2093 "Unexpected setcc expansion!");
2094 if (Chain) {
2095 ReplaceValueWith(SDValue(N, 0), NewLHS);
2096 ReplaceValueWith(SDValue(N, 1), Chain);
2097 return SDValue();
2098 }
2099 return NewLHS;
2100}
2101
2102SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2103 if (ISD::isNormalStore(N))
2104 return ExpandOp_NormalStore(N, OpNo);
2105
2106 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2107 assert(OpNo == 1 && "Can only expand the stored value so far");
2108 StoreSDNode *ST = cast<StoreSDNode>(N);
2109
2110 SDValue Chain = ST->getChain();
2111 SDValue Ptr = ST->getBasePtr();
2112
2113 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2114 ST->getValue().getValueType());
2115 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2116 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2117 (void)NVT;
2118
2119 SDValue Lo, Hi;
2120 GetExpandedOp(ST->getValue(), Lo, Hi);
2121
2122 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2123 ST->getMemoryVT(), ST->getMemOperand());
2124}
2125
2126SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2127 EVT RVT = N->getValueType(0);
2128 EVT RetVT = N->getOperand(0).getValueType();
2130 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2131 RTLIB::LROUND_F32,
2132 RTLIB::LROUND_F64,
2133 RTLIB::LROUND_F80,
2134 RTLIB::LROUND_F128,
2135 RTLIB::LROUND_PPCF128),
2136 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2137}
2138
2139SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2140 EVT RVT = N->getValueType(0);
2141 EVT RetVT = N->getOperand(0).getValueType();
2143 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2144 RTLIB::LLROUND_F32,
2145 RTLIB::LLROUND_F64,
2146 RTLIB::LLROUND_F80,
2147 RTLIB::LLROUND_F128,
2148 RTLIB::LLROUND_PPCF128),
2149 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2150}
2151
2152SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2153 EVT RVT = N->getValueType(0);
2154 EVT RetVT = N->getOperand(0).getValueType();
2156 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2157 RTLIB::LRINT_F32,
2158 RTLIB::LRINT_F64,
2159 RTLIB::LRINT_F80,
2160 RTLIB::LRINT_F128,
2161 RTLIB::LRINT_PPCF128),
2162 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2163}
2164
2165SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2166 EVT RVT = N->getValueType(0);
2167 EVT RetVT = N->getOperand(0).getValueType();
2169 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2170 RTLIB::LLRINT_F32,
2171 RTLIB::LLRINT_F64,
2172 RTLIB::LLRINT_F80,
2173 RTLIB::LLRINT_F128,
2174 RTLIB::LLRINT_PPCF128),
2175 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2176}
2177
2178//===----------------------------------------------------------------------===//
2179// Float Operand Promotion
2180//===----------------------------------------------------------------------===//
2181//
2182
2184 if (OpVT == MVT::f16) {
2185 return ISD::FP16_TO_FP;
2186 } else if (RetVT == MVT::f16) {
2187 return ISD::FP_TO_FP16;
2188 } else if (OpVT == MVT::bf16) {
2189 return ISD::BF16_TO_FP;
2190 } else if (RetVT == MVT::bf16) {
2191 return ISD::FP_TO_BF16;
2192 }
2193
2194 report_fatal_error("Attempt at an invalid promotion-related conversion");
2195}
2196
2198 if (OpVT == MVT::f16)
2200
2201 if (RetVT == MVT::f16)
2203
2204 if (OpVT == MVT::bf16)
2206
2207 if (RetVT == MVT::bf16)
2209
2210 report_fatal_error("Attempt at an invalid promotion-related conversion");
2211}
2212
2213bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2214 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2215 SDValue R = SDValue();
2216
2217 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2218 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2219 return false;
2220 }
2221
2222 // Nodes that use a promotion-requiring floating point operand, but doesn't
2223 // produce a promotion-requiring floating point result, need to be legalized
2224 // to use the promoted float operand. Nodes that produce at least one
2225 // promotion-requiring floating point result have their operands legalized as
2226 // a part of PromoteFloatResult.
2227 // clang-format off
2228 switch (N->getOpcode()) {
2229 default:
2230 #ifndef NDEBUG
2231 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2232 N->dump(&DAG); dbgs() << "\n";
2233 #endif
2234 report_fatal_error("Do not know how to promote this operator's operand!");
2235
2236 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2237 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2238 case ISD::FP_TO_SINT:
2239 case ISD::FP_TO_UINT:
2240 case ISD::LRINT:
2241 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2244 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2245 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2247 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2248 break;
2249 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2250 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2251 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2252 }
2253 // clang-format on
2254
2255 if (R.getNode())
2256 ReplaceValueWith(SDValue(N, 0), R);
2257 return false;
2258}
2259
2260SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2261 SDValue Op = N->getOperand(0);
2262 EVT OpVT = Op->getValueType(0);
2263
2264 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2265 EVT PromotedVT = Promoted->getValueType(0);
2266
2267 // Convert the promoted float value to the desired IVT.
2268 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2269 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2270 IVT, Promoted);
2271 // The final result type might not be an scalar so we need a bitcast. The
2272 // bitcast will be further legalized if needed.
2273 return DAG.getBitcast(N->getValueType(0), Convert);
2274}
2275
2276// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2277// PromoteFloatRes_FCOPYSIGN.
2278SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2279 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2280 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2281
2282 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2283 N->getOperand(0), Op1);
2284}
2285
2286// Convert the promoted float value to the desired integer type
2287SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2288 SDValue Op = GetPromotedFloat(N->getOperand(0));
2289 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2290}
2291
2292SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2293 unsigned OpNo) {
2294 SDValue Op = GetPromotedFloat(N->getOperand(0));
2295 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2296 N->getOperand(1));
2297}
2298
2299SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2300 SDValue Op = GetPromotedFloat(N->getOperand(0));
2301 EVT VT = N->getValueType(0);
2302
2303 // Desired VT is same as promoted type. Use promoted float directly.
2304 if (VT == Op->getValueType(0))
2305 return Op;
2306
2307 // Else, extend the promoted float value to the desired VT.
2308 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2309}
2310
2311SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2312 unsigned OpNo) {
2313 assert(OpNo == 1 && "Promoting unpromotable operand");
2314
2315 SDValue Op = GetPromotedFloat(N->getOperand(1));
2316 EVT VT = N->getValueType(0);
2317
2318 // Desired VT is same as promoted type. Use promoted float directly.
2319 if (VT == Op->getValueType(0)) {
2320 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2321 return Op;
2322 }
2323
2324 // Else, extend the promoted float value to the desired VT.
2325 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2326 N->getOperand(0), Op);
2327 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2328 return Res;
2329}
2330
2331// Promote the float operands used for comparison. The true- and false-
2332// operands have the same type as the result and are promoted, if needed, by
2333// PromoteFloatRes_SELECT_CC
2334SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2335 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2336 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2337
2338 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2339 LHS, RHS, N->getOperand(2), N->getOperand(3),
2340 N->getOperand(4));
2341}
2342
2343// Construct a SETCC that compares the promoted values and sets the conditional
2344// code.
2345SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2346 EVT VT = N->getValueType(0);
2347 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2348 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2349 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2350
2351 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2352
2353}
2354
2355// Lower the promoted Float down to the integer value of same size and construct
2356// a STORE of the integer value.
2357SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2358 StoreSDNode *ST = cast<StoreSDNode>(N);
2359 SDValue Val = ST->getValue();
2360 SDLoc DL(N);
2361
2362 SDValue Promoted = GetPromotedFloat(Val);
2363 EVT VT = ST->getOperand(1).getValueType();
2364 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2365
2366 SDValue NewVal;
2367 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2368 IVT, Promoted);
2369
2370 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2371 ST->getMemOperand());
2372}
2373
2374//===----------------------------------------------------------------------===//
2375// Float Result Promotion
2376//===----------------------------------------------------------------------===//
2377
2378void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2379 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2380 SDValue R = SDValue();
2381
2382 // See if the target wants to custom expand this node.
2383 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2384 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2385 return;
2386 }
2387
2388 switch (N->getOpcode()) {
2389 // These opcodes cannot appear if promotion of FP16 is done in the backend
2390 // instead of Clang
2391 case ISD::FP16_TO_FP:
2392 case ISD::FP_TO_FP16:
2393 default:
2394#ifndef NDEBUG
2395 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2396 N->dump(&DAG); dbgs() << "\n";
2397#endif
2398 report_fatal_error("Do not know how to promote this operator's result!");
2399
2400 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2401 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2403 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2404 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2405
2406 // Unary FP Operations
2407 case ISD::FABS:
2408 case ISD::FCBRT:
2409 case ISD::FCEIL:
2410 case ISD::FCOS:
2411 case ISD::FEXP:
2412 case ISD::FEXP2:
2413 case ISD::FEXP10:
2414 case ISD::FFLOOR:
2415 case ISD::FLOG:
2416 case ISD::FLOG2:
2417 case ISD::FLOG10:
2418 case ISD::FNEARBYINT:
2419 case ISD::FNEG:
2420 case ISD::FRINT:
2421 case ISD::FROUND:
2422 case ISD::FROUNDEVEN:
2423 case ISD::FSIN:
2424 case ISD::FSQRT:
2425 case ISD::FTRUNC:
2426 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2427
2428 // Binary FP Operations
2429 case ISD::FADD:
2430 case ISD::FDIV:
2431 case ISD::FMAXIMUM:
2432 case ISD::FMINIMUM:
2433 case ISD::FMAXNUM:
2434 case ISD::FMINNUM:
2435 case ISD::FMUL:
2436 case ISD::FPOW:
2437 case ISD::FREM:
2438 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2439
2440 case ISD::FMA: // FMA is same as FMAD
2441 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2442
2443 case ISD::FPOWI:
2444 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2445 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2446
2447 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2449 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2450 break;
2451 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2452 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2453 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2454
2455 case ISD::SINT_TO_FP:
2456 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2457 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2458 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2465 R = PromoteFloatRes_VECREDUCE(N);
2466 break;
2469 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2470 break;
2471 }
2472
2473 if (R.getNode())
2474 SetPromotedFloat(SDValue(N, ResNo), R);
2475}
2476
2477// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2478// At this point, it is not possible to determine if the bitcast value is
2479// eventually stored to memory or promoted to f32 or promoted to a floating
2480// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2481// STORE promotion handlers.
2482SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2483 EVT VT = N->getValueType(0);
2484 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2485 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2486 // bitcast will be legalized further if necessary.
2487 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2488 N->getOperand(0).getValueType().getSizeInBits());
2489 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2490 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2491}
2492
2493SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2494 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2495 EVT VT = N->getValueType(0);
2496 SDLoc DL(N);
2497
2498 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2499 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2500 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2501 IVT);
2502
2503 // Convert the Constant to the desired FP type
2504 // FIXME We might be able to do the conversion during compilation and get rid
2505 // of it from the object code
2506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2507 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2508}
2509
2510// If the Index operand is a constant, try to redirect the extract operation to
2511// the correct legalized vector. If not, bit-convert the input vector to
2512// equivalent integer vector. Extract the element as an (bit-cast) integer
2513// value and convert it to the promoted type.
2514SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2515 SDLoc DL(N);
2516
2517 // If the index is constant, try to extract the value from the legalized
2518 // vector type.
2519 if (isa<ConstantSDNode>(N->getOperand(1))) {
2520 SDValue Vec = N->getOperand(0);
2521 SDValue Idx = N->getOperand(1);
2522 EVT VecVT = Vec->getValueType(0);
2523 EVT EltVT = VecVT.getVectorElementType();
2524
2525 uint64_t IdxVal = Idx->getAsZExtVal();
2526
2527 switch (getTypeAction(VecVT)) {
2528 default: break;
2530 SDValue Res = GetScalarizedVector(N->getOperand(0));
2531 ReplaceValueWith(SDValue(N, 0), Res);
2532 return SDValue();
2533 }
2535 Vec = GetWidenedVector(Vec);
2536 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2537 ReplaceValueWith(SDValue(N, 0), Res);
2538 return SDValue();
2539 }
2541 SDValue Lo, Hi;
2542 GetSplitVector(Vec, Lo, Hi);
2543
2544 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2545 SDValue Res;
2546 if (IdxVal < LoElts)
2547 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2548 else
2549 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2550 DAG.getConstant(IdxVal - LoElts, DL,
2551 Idx.getValueType()));
2552 ReplaceValueWith(SDValue(N, 0), Res);
2553 return SDValue();
2554 }
2555
2556 }
2557 }
2558
2559 // Bit-convert the input vector to the equivalent integer vector
2560 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2561 EVT IVT = NewOp.getValueType().getVectorElementType();
2562
2563 // Extract the element as an (bit-cast) integer value
2564 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2565 NewOp, N->getOperand(1));
2566
2567 // Convert the element to the desired FP type
2568 EVT VT = N->getValueType(0);
2569 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2570 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2571}
2572
2573// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2574// needs promotion, so does the argument X. Note that Y, if needed, will be
2575// handled during operand promotion.
2576SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2577 EVT VT = N->getValueType(0);
2578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2579 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2580
2581 SDValue Op1 = N->getOperand(1);
2582
2583 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2584}
2585
2586// Unary operation where the result and the operand have PromoteFloat type
2587// action. Construct a new SDNode with the promoted float value of the old
2588// operand.
2589SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2590 EVT VT = N->getValueType(0);
2591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2592 SDValue Op = GetPromotedFloat(N->getOperand(0));
2593
2594 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2595}
2596
2597// Binary operations where the result and both operands have PromoteFloat type
2598// action. Construct a new SDNode with the promoted float values of the old
2599// operands.
2600SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2601 EVT VT = N->getValueType(0);
2602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2603 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2604 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2605 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2606}
2607
2608SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2609 EVT VT = N->getValueType(0);
2610 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2611 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2612 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2613 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2614
2615 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2616}
2617
2618// Promote the Float (first) operand and retain the Integer (second) operand
2619SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2620 EVT VT = N->getValueType(0);
2621 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2622 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2623 SDValue Op1 = N->getOperand(1);
2624
2625 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2626}
2627
2628SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2629 EVT VT = N->getValueType(0);
2630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2631 SDValue Op = GetPromotedFloat(N->getOperand(0));
2632 SDValue Res =
2633 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2634
2635 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2636 return Res;
2637}
2638
2639// Explicit operation to reduce precision. Reduce the value to half precision
2640// and promote it back to the legal type.
2641SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2642 SDLoc DL(N);
2643
2644 SDValue Op = N->getOperand(0);
2645 EVT VT = N->getValueType(0);
2646 EVT OpVT = Op->getValueType(0);
2647 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2648 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2649
2650 // Round promoted float to desired precision
2651 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2652 // Promote it back to the legal output type
2653 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2654}
2655
2656// Explicit operation to reduce precision. Reduce the value to half precision
2657// and promote it back to the legal type.
2658SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2659 SDLoc DL(N);
2660
2661 SDValue Chain = N->getOperand(0);
2662 SDValue Op = N->getOperand(1);
2663 EVT VT = N->getValueType(0);
2664 EVT OpVT = Op->getValueType(0);
2665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2666 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2667
2668 // Round promoted float to desired precision
2669 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2670 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2671 // Promote it back to the legal output type
2672 SDValue Res =
2673 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2674 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2675 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2676 return Res;
2677}
2678
2679SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2680 LoadSDNode *L = cast<LoadSDNode>(N);
2681 EVT VT = N->getValueType(0);
2682
2683 // Load the value as an integer value with the same number of bits.
2684 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2685 SDValue newL = DAG.getLoad(
2686 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2687 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2688 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2689 // Legalize the chain result by replacing uses of the old value chain with the
2690 // new one
2691 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2692
2693 // Convert the integer value to the desired FP type
2694 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2695 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2696}
2697
2698// Construct a new SELECT node with the promoted true- and false- values.
2699SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2700 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2701 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2702
2703 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2704 N->getOperand(0), TrueVal, FalseVal);
2705}
2706
2707// Construct a new SELECT_CC node with the promoted true- and false- values.
2708// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2709SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2710 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2711 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2712
2713 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2714 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2715 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2716}
2717
2718// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2719// float type.
2720SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2721 SDLoc DL(N);
2722 EVT VT = N->getValueType(0);
2723 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2724 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2725 // Round the value to the desired precision (that of the source type).
2726 return DAG.getNode(
2727 ISD::FP_EXTEND, DL, NVT,
2728 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2729 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2730}
2731
2732SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2733 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2734 N->getValueType(0)));
2735}
2736
2737SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2738 // Expand and promote recursively.
2739 // TODO: This is non-optimal, but dealing with the concurrently happening
2740 // vector-legalization is non-trivial. We could do something similar to
2741 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2742 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2743 return SDValue();
2744}
2745
2746SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2747 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2748 return SDValue();
2749}
2750
2751SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2752 EVT VT = N->getValueType(0);
2753
2754 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2755 SDLoc SL(N);
2756
2757 SDValue CastVal = BitConvertToInteger(AM->getVal());
2758 EVT CastVT = CastVal.getValueType();
2759
2760 SDValue NewAtomic
2761 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2762 DAG.getVTList(CastVT, MVT::Other),
2763 { AM->getChain(), AM->getBasePtr(), CastVal },
2764 AM->getMemOperand());
2765
2766 SDValue Result = NewAtomic;
2767
2768 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2769 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2770 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2771 NewAtomic);
2772 }
2773
2774 // Legalize the chain result by replacing uses of the old value chain with the
2775 // new one
2776 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2777
2778 return Result;
2779
2780}
2781
2782//===----------------------------------------------------------------------===//
2783// Half Result Soft Promotion
2784//===----------------------------------------------------------------------===//
2785
2786void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2787 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2788 N->dump(&DAG));
2789 SDValue R = SDValue();
2790
2791 // See if the target wants to custom expand this node.
2792 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2793 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2794 return;
2795 }
2796
2797 switch (N->getOpcode()) {
2798 default:
2799#ifndef NDEBUG
2800 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2801 N->dump(&DAG); dbgs() << "\n";
2802#endif
2803 report_fatal_error("Do not know how to soft promote this operator's "
2804 "result!");
2805
2806 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2807 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2809 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2810 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2812 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2813
2814 // Unary FP Operations
2815 case ISD::FABS:
2816 case ISD::FCBRT:
2817 case ISD::FCEIL:
2818 case ISD::FCOS:
2819 case ISD::FEXP:
2820 case ISD::FEXP2:
2821 case ISD::FEXP10:
2822 case ISD::FFLOOR:
2823 case ISD::FLOG:
2824 case ISD::FLOG2:
2825 case ISD::FLOG10:
2826 case ISD::FNEARBYINT:
2827 case ISD::FNEG:
2828 case ISD::FREEZE:
2829 case ISD::FRINT:
2830 case ISD::FROUND:
2831 case ISD::FROUNDEVEN:
2832 case ISD::FSIN:
2833 case ISD::FSQRT:
2834 case ISD::FTRUNC:
2835 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2836
2837 // Binary FP Operations
2838 case ISD::FADD:
2839 case ISD::FDIV:
2840 case ISD::FMAXIMUM:
2841 case ISD::FMINIMUM:
2842 case ISD::FMAXNUM:
2843 case ISD::FMINNUM:
2844 case ISD::FMUL:
2845 case ISD::FPOW:
2846 case ISD::FREM:
2847 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2848
2849 case ISD::FMA: // FMA is same as FMAD
2850 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2851
2852 case ISD::FPOWI:
2853 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2854
2855 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2856
2857 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2858 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2859 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2860 case ISD::SINT_TO_FP:
2861 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2862 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2863 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2870 R = SoftPromoteHalfRes_VECREDUCE(N);
2871 break;
2874 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2875 break;
2876 }
2877
2878 if (R.getNode())
2879 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2880}
2881
2882SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2883 return BitConvertToInteger(N->getOperand(0));
2884}
2885
2886SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2887 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2888
2889 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2890 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2891 MVT::i16);
2892}
2893
2894SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2895 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2897 NewOp.getValueType().getVectorElementType(), NewOp,
2898 N->getOperand(1));
2899}
2900
2901SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2902 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2903 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2904 SDLoc dl(N);
2905
2906 EVT LVT = LHS.getValueType();
2907 EVT RVT = RHS.getValueType();
2908
2909 unsigned LSize = LVT.getSizeInBits();
2910 unsigned RSize = RVT.getSizeInBits();
2911
2912 // First get the sign bit of second operand.
2913 SDValue SignBit = DAG.getNode(
2914 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2915 DAG.getConstant(RSize - 1, dl,
2916 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2917 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2918
2919 // Shift right or sign-extend it if the two operands have different types.
2920 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2921 if (SizeDiff > 0) {
2922 SignBit =
2923 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2924 DAG.getConstant(SizeDiff, dl,
2925 TLI.getShiftAmountTy(SignBit.getValueType(),
2926 DAG.getDataLayout())));
2927 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2928 } else if (SizeDiff < 0) {
2929 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2930 SignBit =
2931 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2932 DAG.getConstant(-SizeDiff, dl,
2933 TLI.getShiftAmountTy(SignBit.getValueType(),
2934 DAG.getDataLayout())));
2935 }
2936
2937 // Clear the sign bit of the first operand.
2938 SDValue Mask = DAG.getNode(
2939 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2940 DAG.getConstant(LSize - 1, dl,
2941 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2942 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2943 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2944
2945 // Or the value with the sign bit.
2946 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2947}
2948
2949SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2950 EVT OVT = N->getValueType(0);
2951 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2952 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2953 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2954 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2955 SDLoc dl(N);
2956
2957 // Promote to the larger FP type.
2958 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2959 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2960 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2961 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2962
2963 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2964
2965 // Convert back to FP16 as an integer.
2966 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2967}
2968
2969SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2970 EVT OVT = N->getValueType(0);
2971 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2972 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2973 SDValue Op1 = N->getOperand(1);
2974 SDLoc dl(N);
2975
2976 // Promote to the larger FP type.
2977 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2978
2979 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2980
2981 // Convert back to FP16 as an integer.
2982 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2983}
2984
2985SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2986 EVT OVT = N->getValueType(0);
2987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2988 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2989 SDLoc dl(N);
2990
2991 // Promote to the larger FP type.
2992 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2993
2994 SDValue Res = DAG.getNode(N->getOpcode(), dl,
2995 DAG.getVTList(NVT, N->getValueType(1)), Op);
2996
2997 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2998
2999 // Convert back to FP16 as an integer.
3000 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3001}
3002
3003SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3004 EVT RVT = N->getValueType(0);
3005 EVT SVT = N->getOperand(0).getValueType();
3006
3007 if (N->isStrictFPOpcode()) {
3008 // FIXME: assume we only have two f16 variants for now.
3009 unsigned Opcode;
3010 if (RVT == MVT::f16)
3011 Opcode = ISD::STRICT_FP_TO_FP16;
3012 else if (RVT == MVT::bf16)
3013 Opcode = ISD::STRICT_FP_TO_BF16;
3014 else
3015 llvm_unreachable("unknown half type");
3016 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3017 {N->getOperand(0), N->getOperand(1)});
3018 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3019 return Res;
3020 }
3021
3022 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3023 N->getOperand(0));
3024}
3025
3026SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3027 LoadSDNode *L = cast<LoadSDNode>(N);
3028
3029 // Load the value as an integer value with the same number of bits.
3030 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3031 SDValue NewL =
3032 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3033 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3034 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3035 L->getMemOperand()->getFlags(), L->getAAInfo());
3036 // Legalize the chain result by replacing uses of the old value chain with the
3037 // new one
3038 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3039 return NewL;
3040}
3041
3042SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3043 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3044 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3045 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3046 Op2);
3047}
3048
3049SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3050 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3051 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3052 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3053 N->getOperand(0), N->getOperand(1), Op2, Op3,
3054 N->getOperand(4));
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3058 EVT OVT = N->getValueType(0);
3059 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3060 SDLoc dl(N);
3061
3062 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3063
3064 // Round the value to the softened type.
3065 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3066}
3067
3068SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3069 return DAG.getUNDEF(MVT::i16);
3070}
3071
3072SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3073 EVT OVT = N->getValueType(0);
3074 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3075 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3076 SDLoc dl(N);
3077
3078 // Promote to the larger FP type.
3079 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3080
3081 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3082
3083 // Convert back to FP16 as an integer.
3084 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3085}
3086
3087SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3088 EVT OVT = N->getValueType(0);
3089 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3090 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3091 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3092 SDLoc dl(N);
3093
3094 // Promote to the larger FP type.
3095 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3096 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3097 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3098
3099 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3100
3101 // Convert back to FP16 as an integer.
3102 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3103}
3104
3105SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3106 // Expand and soften recursively.
3107 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3108 return SDValue();
3109}
3110
3111SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3112 // Expand and soften.
3113 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3114 return SDValue();
3115}
3116
3117//===----------------------------------------------------------------------===//
3118// Half Operand Soft Promotion
3119//===----------------------------------------------------------------------===//
3120
3121bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3122 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3123 N->dump(&DAG));
3124 SDValue Res = SDValue();
3125
3126 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3127 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3128 return false;
3129 }
3130
3131 // Nodes that use a promotion-requiring floating point operand, but doesn't
3132 // produce a soft promotion-requiring floating point result, need to be
3133 // legalized to use the soft promoted float operand. Nodes that produce at
3134 // least one soft promotion-requiring floating point result have their
3135 // operands legalized as a part of PromoteFloatResult.
3136 switch (N->getOpcode()) {
3137 default:
3138 #ifndef NDEBUG
3139 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3140 N->dump(&DAG); dbgs() << "\n";
3141 #endif
3142 report_fatal_error("Do not know how to soft promote this operator's "
3143 "operand!");
3144
3145 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3146 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3147 case ISD::FP_TO_SINT:
3148 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3151 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3153 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3154 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3155 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3156 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3157 case ISD::STACKMAP:
3158 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3159 break;
3160 case ISD::PATCHPOINT:
3161 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3162 break;
3163 }
3164
3165 if (!Res.getNode())
3166 return false;
3167
3168 assert(Res.getNode() != N && "Expected a new node!");
3169
3170 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3171 "Invalid operand expansion");
3172
3173 ReplaceValueWith(SDValue(N, 0), Res);
3174 return false;
3175}
3176
3177SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3178 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3179
3180 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3181}
3182
3183SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3184 unsigned OpNo) {
3185 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3186 SDValue Op1 = N->getOperand(1);
3187 EVT RVT = Op1.getValueType();
3188 SDLoc dl(N);
3189
3190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3191
3192 Op1 = GetSoftPromotedHalf(Op1);
3193 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3194
3195 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3196 Op1);
3197}
3198
3199SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3200 EVT RVT = N->getValueType(0);
3201 bool IsStrict = N->isStrictFPOpcode();
3202 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3203 EVT SVT = Op.getValueType();
3204 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3205
3206 if (IsStrict) {
3207 unsigned Opcode;
3208 if (SVT == MVT::f16)
3209 Opcode = ISD::STRICT_FP16_TO_FP;
3210 else if (SVT == MVT::bf16)
3211 Opcode = ISD::STRICT_BF16_TO_FP;
3212 else
3213 llvm_unreachable("unknown half type");
3214 SDValue Res =
3215 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3216 {N->getOperand(0), Op});
3217 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3218 ReplaceValueWith(SDValue(N, 0), Res);
3219 return SDValue();
3220 }
3221
3222 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3223}
3224
3225SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3226 EVT RVT = N->getValueType(0);
3227 SDValue Op = N->getOperand(0);
3228 EVT SVT = Op.getValueType();
3229 SDLoc dl(N);
3230
3231 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3232
3233 Op = GetSoftPromotedHalf(Op);
3234
3235 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3236
3237 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3238}
3239
3240SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3241 EVT RVT = N->getValueType(0);
3242 SDValue Op = N->getOperand(0);
3243 EVT SVT = Op.getValueType();
3244 SDLoc dl(N);
3245
3246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3247
3248 Op = GetSoftPromotedHalf(Op);
3249
3250 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3251
3252 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3253 N->getOperand(1));
3254}
3255
3256SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3257 unsigned OpNo) {
3258 assert(OpNo == 0 && "Can only soften the comparison values");
3259 SDValue Op0 = N->getOperand(0);
3260 SDValue Op1 = N->getOperand(1);
3261 SDLoc dl(N);
3262
3263 EVT SVT = Op0.getValueType();
3264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3265
3266 Op0 = GetSoftPromotedHalf(Op0);
3267 Op1 = GetSoftPromotedHalf(Op1);
3268
3269 // Promote to the larger FP type.
3270 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3271 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3272 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3273
3274 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3275 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3276}
3277
3278SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3279 SDValue Op0 = N->getOperand(0);
3280 SDValue Op1 = N->getOperand(1);
3281 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3282 SDLoc dl(N);
3283
3284 EVT SVT = Op0.getValueType();
3285 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3286
3287 Op0 = GetSoftPromotedHalf(Op0);
3288 Op1 = GetSoftPromotedHalf(Op1);
3289
3290 // Promote to the larger FP type.
3291 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3292 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3293 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3294
3295 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3296}
3297
3298SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3299 assert(OpNo == 1 && "Can only soften the stored value!");
3300 StoreSDNode *ST = cast<StoreSDNode>(N);
3301 SDValue Val = ST->getValue();
3302 SDLoc dl(N);
3303
3304 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3305 SDValue Promoted = GetSoftPromotedHalf(Val);
3306 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3307 ST->getMemOperand());
3308}
3309
3310SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3311 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3312 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3313 SDValue Op = N->getOperand(OpNo);
3314 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3316 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3317
3318 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3319 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3320
3321 return SDValue(); // Signal that we replaced the node ourselves.
3322}
3323
3324SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3325 unsigned OpNo) {
3326 assert(OpNo >= 7);
3327 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3328 SDValue Op = N->getOperand(OpNo);
3329 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3331 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3332
3333 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3334 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3335
3336 return SDValue(); // Signal that we replaced the node ourselves.
3337}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
DXIL Intrinsic Expansion
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:471
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:478
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:468
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:484
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:553
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ ConstantFP
Definition: ISDOpcodes.h:77
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:924
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
DWARFExpression::Operation Op
#define N
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)