LLVM 17.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 llvm_unreachable("Do not know how to soften the result of this operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
93 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
95 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
97 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
98 case ISD::STRICT_FMA:
99 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
100 case ISD::STRICT_FMUL:
101 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
103 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
104 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
106 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
108 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
109 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
110 case ISD::STRICT_FPOW:
111 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
113 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
114 case ISD::STRICT_FREM:
115 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
117 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
119 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
121 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
122 case ISD::STRICT_FSIN:
123 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
125 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
126 case ISD::STRICT_FSUB:
127 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
129 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
130 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
131 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
132 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
133 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
134 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
137 case ISD::SINT_TO_FP:
138 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
139 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
140 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
145 R = SoftenFloatRes_VECREDUCE(N);
146 break;
149 R = SoftenFloatRes_VECREDUCE_SEQ(N);
150 break;
151 }
152
153 // If R is null, the sub-method took care of registering the result.
154 if (R.getNode()) {
155 assert(R.getNode() != N);
156 SetSoftenedFloat(SDValue(N, ResNo), R);
157 }
158}
159
160SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
161 bool IsStrict = N->isStrictFPOpcode();
162 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
163 unsigned Offset = IsStrict ? 1 : 0;
164 assert(N->getNumOperands() == (1 + Offset) &&
165 "Unexpected number of operands!");
166 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
167 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
169 EVT OpVT = N->getOperand(0 + Offset).getValueType();
170 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
171 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
172 CallOptions, SDLoc(N),
173 Chain);
174 if (IsStrict)
175 ReplaceValueWith(SDValue(N, 1), Tmp.second);
176 return Tmp.first;
177}
178
179SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
180 bool IsStrict = N->isStrictFPOpcode();
181 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
182 unsigned Offset = IsStrict ? 1 : 0;
183 assert(N->getNumOperands() == (2 + Offset) &&
184 "Unexpected number of operands!");
185 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
186 GetSoftenedFloat(N->getOperand(1 + Offset)) };
187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
189 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
190 N->getOperand(1 + Offset).getValueType() };
191 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
192 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
193 CallOptions, SDLoc(N),
194 Chain);
195 if (IsStrict)
196 ReplaceValueWith(SDValue(N, 1), Tmp.second);
197 return Tmp.first;
198}
199
200SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
201 return BitConvertToInteger(N->getOperand(0));
202}
203
204SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
205 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
206 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
207 GetSoftenedFloat(N->getOperand(0)));
208}
209
210SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
211 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
212 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
213 GetSoftenedFloat(N->getOperand(0)));
214 return NewFence;
215}
216
217SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
218 unsigned ResNo) {
219 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
220 return BitConvertToInteger(Op);
221}
222
223SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
224 // Convert the inputs to integers, and build a new pair out of them.
225 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
227 N->getValueType(0)),
228 BitConvertToInteger(N->getOperand(0)),
229 BitConvertToInteger(N->getOperand(1)));
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
233 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
234 // In ppcf128, the high 64 bits are always first in memory regardless
235 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
236 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
237 // way. However, APInt's are serialized in an Endian-sensitive fashion,
238 // so on big-Endian targets, the two doubles are output in the wrong
239 // order. Fix this by manually flipping the order of the high 64 bits
240 // and the low 64 bits here.
241 if (DAG.getDataLayout().isBigEndian() &&
243 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
245 APInt Val(128, words);
246 return DAG.getConstant(Val, SDLoc(CN),
248 CN->getValueType(0)));
249 } else {
250 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
252 CN->getValueType(0)));
253 }
254}
255
256SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
257 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
260 NewOp, N->getOperand(1));
261}
262
263SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
265 unsigned Size = NVT.getSizeInBits();
266
267 // Mask = ~(1 << (Size-1))
269 API.clearBit(Size - 1);
270 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
271 SDValue Op = GetSoftenedFloat(N->getOperand(0));
272 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
273}
274
275SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
276 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
277 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
278 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
279 RTLIB::FMIN_F32,
280 RTLIB::FMIN_F64,
281 RTLIB::FMIN_F80,
282 RTLIB::FMIN_F128,
283 RTLIB::FMIN_PPCF128));
284}
285
286SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
287 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
288 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
289 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
290 RTLIB::FMAX_F32,
291 RTLIB::FMAX_F64,
292 RTLIB::FMAX_F80,
293 RTLIB::FMAX_F128,
294 RTLIB::FMAX_PPCF128));
295}
296
297SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
299 RTLIB::ADD_F32,
300 RTLIB::ADD_F64,
301 RTLIB::ADD_F80,
302 RTLIB::ADD_F128,
303 RTLIB::ADD_PPCF128));
304}
305
306SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
307 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::CBRT_F32,
309 RTLIB::CBRT_F64,
310 RTLIB::CBRT_F80,
311 RTLIB::CBRT_F128,
312 RTLIB::CBRT_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
317 RTLIB::CEIL_F32,
318 RTLIB::CEIL_F64,
319 RTLIB::CEIL_F80,
320 RTLIB::CEIL_F128,
321 RTLIB::CEIL_PPCF128));
322}
323
324SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
325 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
326 SDValue RHS = BitConvertToInteger(N->getOperand(1));
327 SDLoc dl(N);
328
329 EVT LVT = LHS.getValueType();
330 EVT RVT = RHS.getValueType();
331
332 unsigned LSize = LVT.getSizeInBits();
333 unsigned RSize = RVT.getSizeInBits();
334
335 // First get the sign bit of second operand.
336 SDValue SignBit = DAG.getNode(
337 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
338 DAG.getConstant(RSize - 1, dl,
339 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
340 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
341
342 // Shift right or sign-extend it if the two operands have different types.
343 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
344 if (SizeDiff > 0) {
345 SignBit =
346 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
347 DAG.getConstant(SizeDiff, dl,
348 TLI.getShiftAmountTy(SignBit.getValueType(),
349 DAG.getDataLayout())));
350 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
351 } else if (SizeDiff < 0) {
352 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
353 SignBit =
354 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
355 DAG.getConstant(-SizeDiff, dl,
356 TLI.getShiftAmountTy(SignBit.getValueType(),
357 DAG.getDataLayout())));
358 }
359
360 // Clear the sign bit of the first operand.
361 SDValue Mask = DAG.getNode(
362 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
363 DAG.getConstant(LSize - 1, dl,
364 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
365 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
366 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
367
368 // Or the value with the sign bit.
369 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
370}
371
372SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
373 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
374 RTLIB::COS_F32,
375 RTLIB::COS_F64,
376 RTLIB::COS_F80,
377 RTLIB::COS_F128,
378 RTLIB::COS_PPCF128));
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
382 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
383 RTLIB::DIV_F32,
384 RTLIB::DIV_F64,
385 RTLIB::DIV_F80,
386 RTLIB::DIV_F128,
387 RTLIB::DIV_PPCF128));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
391 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
392 RTLIB::EXP_F32,
393 RTLIB::EXP_F64,
394 RTLIB::EXP_F80,
395 RTLIB::EXP_F128,
396 RTLIB::EXP_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
401 RTLIB::EXP2_F32,
402 RTLIB::EXP2_F64,
403 RTLIB::EXP2_F80,
404 RTLIB::EXP2_F128,
405 RTLIB::EXP2_PPCF128));
406}
407
408SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
410 RTLIB::FLOOR_F32,
411 RTLIB::FLOOR_F64,
412 RTLIB::FLOOR_F80,
413 RTLIB::FLOOR_F128,
414 RTLIB::FLOOR_PPCF128));
415}
416
417SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
418 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
419 RTLIB::LOG_F32,
420 RTLIB::LOG_F64,
421 RTLIB::LOG_F80,
422 RTLIB::LOG_F128,
423 RTLIB::LOG_PPCF128));
424}
425
426SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
427 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
428 RTLIB::LOG2_F32,
429 RTLIB::LOG2_F64,
430 RTLIB::LOG2_F80,
431 RTLIB::LOG2_F128,
432 RTLIB::LOG2_PPCF128));
433}
434
435SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
436 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
437 RTLIB::LOG10_F32,
438 RTLIB::LOG10_F64,
439 RTLIB::LOG10_F80,
440 RTLIB::LOG10_F128,
441 RTLIB::LOG10_PPCF128));
442}
443
444SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
445 bool IsStrict = N->isStrictFPOpcode();
446 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
447 unsigned Offset = IsStrict ? 1 : 0;
448 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
449 GetSoftenedFloat(N->getOperand(1 + Offset)),
450 GetSoftenedFloat(N->getOperand(2 + Offset)) };
451 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
453 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
454 N->getOperand(1 + Offset).getValueType(),
455 N->getOperand(2 + Offset).getValueType() };
456 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
458 GetFPLibCall(N->getValueType(0),
459 RTLIB::FMA_F32,
460 RTLIB::FMA_F64,
461 RTLIB::FMA_F80,
462 RTLIB::FMA_F128,
463 RTLIB::FMA_PPCF128),
464 NVT, Ops, CallOptions, SDLoc(N), Chain);
465 if (IsStrict)
466 ReplaceValueWith(SDValue(N, 1), Tmp.second);
467 return Tmp.first;
468}
469
470SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
471 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
472 RTLIB::MUL_F32,
473 RTLIB::MUL_F64,
474 RTLIB::MUL_F80,
475 RTLIB::MUL_F128,
476 RTLIB::MUL_PPCF128));
477}
478
479SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
480 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
481 RTLIB::NEARBYINT_F32,
482 RTLIB::NEARBYINT_F64,
483 RTLIB::NEARBYINT_F80,
484 RTLIB::NEARBYINT_F128,
485 RTLIB::NEARBYINT_PPCF128));
486}
487
488SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
490 SDLoc dl(N);
491
492 // Expand Y = FNEG(X) -> Y = X ^ sign mask
493 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
494 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
495 DAG.getConstant(SignMask, dl, NVT));
496}
497
498SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
499 bool IsStrict = N->isStrictFPOpcode();
500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
501 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
502
503 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
504
505 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
506 Op = GetPromotedFloat(Op);
507 // If the promotion did the FP_EXTEND to the destination type for us,
508 // there's nothing left to do here.
509 if (Op.getValueType() == N->getValueType(0))
510 return BitConvertToInteger(Op);
511 }
512
513 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's
514 // entirely possible for both f16 and f32 to be legal, so use the fully
515 // hard-float FP_EXTEND rather than FP16_TO_FP.
516 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) {
517 if (IsStrict) {
519 { MVT::f32, MVT::Other }, { Chain, Op });
520 Chain = Op.getValue(1);
521 } else {
522 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
523 }
524 }
525
526 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
527 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
529 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
530 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
531 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
532 CallOptions, SDLoc(N),
533 Chain);
534 if (IsStrict)
535 ReplaceValueWith(SDValue(N, 1), Tmp.second);
536 return Tmp.first;
537}
538
539// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
540// nodes?
541SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
542 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
543 SDValue Op = N->getOperand(0);
545 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
546 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
547 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
548 CallOptions, SDLoc(N)).first;
549 if (N->getValueType(0) == MVT::f32)
550 return Res32;
551
552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
553 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
554 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
555 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
556}
557
558SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
559 bool IsStrict = N->isStrictFPOpcode();
560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
561 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
562 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
563 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
564 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
566 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
567 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
568 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
569 CallOptions, SDLoc(N),
570 Chain);
571 if (IsStrict)
572 ReplaceValueWith(SDValue(N, 1), Tmp.second);
573 return Tmp.first;
574}
575
576SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
577 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
578 RTLIB::POW_F32,
579 RTLIB::POW_F64,
580 RTLIB::POW_F80,
581 RTLIB::POW_F128,
582 RTLIB::POW_PPCF128));
583}
584
585SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
586 bool IsStrict = N->isStrictFPOpcode();
587 unsigned Offset = IsStrict ? 1 : 0;
588 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
589 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
590 "Unsupported power type!");
591 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
592 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
593 if (!TLI.getLibcallName(LC)) {
594 // Some targets don't have a powi libcall; use pow instead.
595 // FIXME: Implement this if some target needs it.
596 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
597 return DAG.getUNDEF(N->getValueType(0));
598 }
599
600 if (DAG.getLibInfo().getIntSize() !=
601 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
602 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
603 // would use the wrong type for the argument.
604 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
605 return DAG.getUNDEF(N->getValueType(0));
606 }
607
608 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
609 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
610 N->getOperand(1 + Offset) };
611 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
613 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
614 N->getOperand(1 + Offset).getValueType() };
615 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
616 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
617 CallOptions, SDLoc(N),
618 Chain);
619 if (IsStrict)
620 ReplaceValueWith(SDValue(N, 1), Tmp.second);
621 return Tmp.first;
622}
623
624SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
625 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
626 RTLIB::REM_F32,
627 RTLIB::REM_F64,
628 RTLIB::REM_F80,
629 RTLIB::REM_F128,
630 RTLIB::REM_PPCF128));
631}
632
633SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
634 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
635 RTLIB::RINT_F32,
636 RTLIB::RINT_F64,
637 RTLIB::RINT_F80,
638 RTLIB::RINT_F128,
639 RTLIB::RINT_PPCF128));
640}
641
642SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
643 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
644 RTLIB::ROUND_F32,
645 RTLIB::ROUND_F64,
646 RTLIB::ROUND_F80,
647 RTLIB::ROUND_F128,
648 RTLIB::ROUND_PPCF128));
649}
650
651SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
652 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
653 RTLIB::ROUNDEVEN_F32,
654 RTLIB::ROUNDEVEN_F64,
655 RTLIB::ROUNDEVEN_F80,
656 RTLIB::ROUNDEVEN_F128,
657 RTLIB::ROUNDEVEN_PPCF128));
658}
659
660SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
661 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
662 RTLIB::SIN_F32,
663 RTLIB::SIN_F64,
664 RTLIB::SIN_F80,
665 RTLIB::SIN_F128,
666 RTLIB::SIN_PPCF128));
667}
668
669SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
670 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
671 RTLIB::SQRT_F32,
672 RTLIB::SQRT_F64,
673 RTLIB::SQRT_F80,
674 RTLIB::SQRT_F128,
675 RTLIB::SQRT_PPCF128));
676}
677
678SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
679 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
680 RTLIB::SUB_F32,
681 RTLIB::SUB_F64,
682 RTLIB::SUB_F80,
683 RTLIB::SUB_F128,
684 RTLIB::SUB_PPCF128));
685}
686
687SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
688 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
689 RTLIB::TRUNC_F32,
690 RTLIB::TRUNC_F64,
691 RTLIB::TRUNC_F80,
692 RTLIB::TRUNC_F128,
693 RTLIB::TRUNC_PPCF128));
694}
695
696SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
697 LoadSDNode *L = cast<LoadSDNode>(N);
698 EVT VT = N->getValueType(0);
699 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
700 SDLoc dl(N);
701
702 auto MMOFlags =
703 L->getMemOperand()->getFlags() &
705 SDValue NewL;
706 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
707 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
708 L->getChain(), L->getBasePtr(), L->getOffset(),
709 L->getPointerInfo(), NVT, L->getOriginalAlign(),
710 MMOFlags, L->getAAInfo());
711 // Legalized the chain result - switch anything that used the old chain to
712 // use the new one.
713 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
714 return NewL;
715 }
716
717 // Do a non-extending load followed by FP_EXTEND.
718 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
719 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
720 L->getPointerInfo(), L->getMemoryVT(),
721 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
722 // Legalized the chain result - switch anything that used the old chain to
723 // use the new one.
724 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
725 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
726 return BitConvertToInteger(ExtendNode);
727}
728
729SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
730 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
731 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
732 return DAG.getSelect(SDLoc(N),
733 LHS.getValueType(), N->getOperand(0), LHS, RHS);
734}
735
736SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
737 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
738 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
739 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
740 LHS.getValueType(), N->getOperand(0),
741 N->getOperand(1), LHS, RHS, N->getOperand(4));
742}
743
744SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
745 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
746 N->getValueType(0)));
747}
748
749SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
750 SDValue Chain = N->getOperand(0); // Get the chain.
751 SDValue Ptr = N->getOperand(1); // Get the pointer.
752 EVT VT = N->getValueType(0);
753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
754 SDLoc dl(N);
755
756 SDValue NewVAARG;
757 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
758 N->getConstantOperandVal(3));
759
760 // Legalized the chain result - switch anything that used the old chain to
761 // use the new one.
762 if (N != NewVAARG.getValue(1).getNode())
763 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
764 return NewVAARG;
765}
766
767SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
768 bool IsStrict = N->isStrictFPOpcode();
769 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
770 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
771 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
772 EVT RVT = N->getValueType(0);
773 EVT NVT = EVT();
774 SDLoc dl(N);
775
776 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
777 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
778 // match. Look for an appropriate libcall.
779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
780 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
781 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
782 NVT = (MVT::SimpleValueType)t;
783 // The source needs to big enough to hold the operand.
784 if (NVT.bitsGE(SVT))
785 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
786 }
787 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
788
789 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
790 // Sign/zero extend the argument if the libcall takes a larger type.
792 NVT, N->getOperand(IsStrict ? 1 : 0));
794 CallOptions.setSExt(Signed);
795 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
796 std::pair<SDValue, SDValue> Tmp =
797 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
798 Op, CallOptions, dl, Chain);
799
800 if (IsStrict)
801 ReplaceValueWith(SDValue(N, 1), Tmp.second);
802 return Tmp.first;
803}
804
805SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
806 // Expand and soften recursively.
807 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
808 return SDValue();
809}
810
811SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
812 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
813 return SDValue();
814}
815
816//===----------------------------------------------------------------------===//
817// Convert Float Operand to Integer
818//===----------------------------------------------------------------------===//
819
820bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
821 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
822 dbgs() << "\n");
823 SDValue Res = SDValue();
824
825 switch (N->getOpcode()) {
826 default:
827#ifndef NDEBUG
828 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
829 N->dump(&DAG); dbgs() << "\n";
830#endif
831 llvm_unreachable("Do not know how to soften this operator's operand!");
832
833 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
834 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
836 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
837 case ISD::FP_TO_BF16:
839 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
842 case ISD::FP_TO_SINT:
843 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
846 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
848 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
850 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
852 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
854 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
855 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
858 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
859 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
860 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
861 }
862
863 // If the result is null, the sub-method took care of registering results etc.
864 if (!Res.getNode()) return false;
865
866 // If the result is N, the sub-method updated N in place. Tell the legalizer
867 // core about this to re-analyze.
868 if (Res.getNode() == N)
869 return true;
870
871 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
872 "Invalid operand softening");
873
874 ReplaceValueWith(SDValue(N, 0), Res);
875 return false;
876}
877
878SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
879 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
880
881 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
885 // We actually deal with the partially-softened FP_TO_FP16 node too, which
886 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
887 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
888 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
889 N->getOpcode() == ISD::FP_TO_BF16 ||
890 N->getOpcode() == ISD::STRICT_FP_ROUND);
891
892 bool IsStrict = N->isStrictFPOpcode();
893 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
894 EVT SVT = Op.getValueType();
895 EVT RVT = N->getValueType(0);
896 EVT FloatRVT = RVT;
897 if (N->getOpcode() == ISD::FP_TO_FP16 ||
898 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
899 FloatRVT = MVT::f16;
900 else if (N->getOpcode() == ISD::FP_TO_BF16)
901 FloatRVT = MVT::bf16;
902
903 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
904 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
905
906 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
907 Op = GetSoftenedFloat(Op);
909 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
910 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
911 CallOptions, SDLoc(N),
912 Chain);
913 if (IsStrict) {
914 ReplaceValueWith(SDValue(N, 1), Tmp.second);
915 ReplaceValueWith(SDValue(N, 0), Tmp.first);
916 return SDValue();
917 }
918 return Tmp.first;
919}
920
921SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
922 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
923 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
924
925 EVT VT = NewLHS.getValueType();
926 NewLHS = GetSoftenedFloat(NewLHS);
927 NewRHS = GetSoftenedFloat(NewRHS);
928 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
929 N->getOperand(2), N->getOperand(3));
930
931 // If softenSetCCOperands returned a scalar, we need to compare the result
932 // against zero to select between true and false values.
933 if (!NewRHS.getNode()) {
934 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
935 CCCode = ISD::SETNE;
936 }
937
938 // Update N to have the operands specified.
939 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
940 DAG.getCondCode(CCCode), NewLHS, NewRHS,
941 N->getOperand(4)),
942 0);
943}
944
945// Even if the result type is legal, no libcall may exactly match. (e.g. We
946// don't have FP-i8 conversions) This helper method looks for an appropriate
947// promoted libcall.
948static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
949 bool Signed) {
950 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
951 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
952 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
953 ++IntVT) {
954 Promoted = (MVT::SimpleValueType)IntVT;
955 // The type needs to big enough to hold the result.
956 if (Promoted.bitsGE(RetVT))
957 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
958 : RTLIB::getFPTOUINT(SrcVT, Promoted);
959 }
960 return LC;
961}
962
963SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
964 bool IsStrict = N->isStrictFPOpcode();
965 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
966 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
967
968 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
969 EVT SVT = Op.getValueType();
970 EVT RVT = N->getValueType(0);
971 EVT NVT = EVT();
972 SDLoc dl(N);
973
974 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
975 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
976 // match, eg. we don't have fp -> i8 conversions.
977 // Look for an appropriate libcall.
978 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
979 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
980 "Unsupported FP_TO_XINT!");
981
982 Op = GetSoftenedFloat(Op);
983 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
985 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
986 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
987 CallOptions, dl, Chain);
988
989 // Truncate the result if the libcall returns a larger type.
990 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
991
992 if (!IsStrict)
993 return Res;
994
995 ReplaceValueWith(SDValue(N, 1), Tmp.second);
996 ReplaceValueWith(SDValue(N, 0), Res);
997 return SDValue();
998}
999
1000SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1001 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1002 return Res;
1003}
1004
1005SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1006 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1007 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1008
1009 EVT VT = NewLHS.getValueType();
1010 NewLHS = GetSoftenedFloat(NewLHS);
1011 NewRHS = GetSoftenedFloat(NewRHS);
1012 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1013 N->getOperand(0), N->getOperand(1));
1014
1015 // If softenSetCCOperands returned a scalar, we need to compare the result
1016 // against zero to select between true and false values.
1017 if (!NewRHS.getNode()) {
1018 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1019 CCCode = ISD::SETNE;
1020 }
1021
1022 // Update N to have the operands specified.
1023 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1024 N->getOperand(2), N->getOperand(3),
1025 DAG.getCondCode(CCCode)),
1026 0);
1027}
1028
1029SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1030 bool IsStrict = N->isStrictFPOpcode();
1031 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1032 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1033 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1034 ISD::CondCode CCCode =
1035 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1036
1037 EVT VT = Op0.getValueType();
1038 SDValue NewLHS = GetSoftenedFloat(Op0);
1039 SDValue NewRHS = GetSoftenedFloat(Op1);
1040 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1041 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1042
1043 // Update N to have the operands specified.
1044 if (NewRHS.getNode()) {
1045 if (IsStrict)
1046 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1047 NewRHS, DAG.getCondCode(CCCode));
1048 else
1049 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1050 DAG.getCondCode(CCCode)), 0);
1051 }
1052
1053 // Otherwise, softenSetCCOperands returned a scalar, use it.
1054 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1055 "Unexpected setcc expansion!");
1056
1057 if (IsStrict) {
1058 ReplaceValueWith(SDValue(N, 0), NewLHS);
1059 ReplaceValueWith(SDValue(N, 1), Chain);
1060 return SDValue();
1061 }
1062 return NewLHS;
1063}
1064
1065SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1066 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1067 assert(OpNo == 1 && "Can only soften the stored value!");
1068 StoreSDNode *ST = cast<StoreSDNode>(N);
1069 SDValue Val = ST->getValue();
1070 SDLoc dl(N);
1071
1072 if (ST->isTruncatingStore())
1073 // Do an FP_ROUND followed by a non-truncating store.
1074 Val = BitConvertToInteger(
1075 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1076 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1077 else
1078 Val = GetSoftenedFloat(Val);
1079
1080 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1081 ST->getMemOperand());
1082}
1083
1084SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1085 SDValue LHS = N->getOperand(0);
1086 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1087 SDLoc dl(N);
1088
1089 EVT LVT = LHS.getValueType();
1090 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1091 EVT RVT = RHS.getValueType();
1092
1093 unsigned LSize = LVT.getSizeInBits();
1094 unsigned RSize = RVT.getSizeInBits();
1095
1096 // Shift right or sign-extend it if the two operands have different types.
1097 int SizeDiff = RSize - LSize;
1098 if (SizeDiff > 0) {
1099 RHS =
1100 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1101 DAG.getConstant(SizeDiff, dl,
1102 TLI.getShiftAmountTy(RHS.getValueType(),
1103 DAG.getDataLayout())));
1104 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1105 } else if (SizeDiff < 0) {
1106 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1107 RHS =
1108 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1109 DAG.getConstant(-SizeDiff, dl,
1110 TLI.getShiftAmountTy(RHS.getValueType(),
1111 DAG.getDataLayout())));
1112 }
1113
1114 RHS = DAG.getBitcast(LVT, RHS);
1115 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1116}
1117
1118SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1119 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1120 bool IsStrict = N->isStrictFPOpcode();
1121 unsigned Offset = IsStrict ? 1 : 0;
1122 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1123 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1125 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1126 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1127 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1128 CallOptions, SDLoc(N),
1129 Chain);
1130 if (IsStrict) {
1131 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1132 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1133 return SDValue();
1134 }
1135
1136 return Tmp.first;
1137}
1138
1139SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1140 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1141 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1142 RTLIB::LROUND_F32,
1143 RTLIB::LROUND_F64,
1144 RTLIB::LROUND_F80,
1145 RTLIB::LROUND_F128,
1146 RTLIB::LROUND_PPCF128));
1147}
1148
1149SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1150 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1151 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1152 RTLIB::LLROUND_F32,
1153 RTLIB::LLROUND_F64,
1154 RTLIB::LLROUND_F80,
1155 RTLIB::LLROUND_F128,
1156 RTLIB::LLROUND_PPCF128));
1157}
1158
1159SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1160 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1161 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1162 RTLIB::LRINT_F32,
1163 RTLIB::LRINT_F64,
1164 RTLIB::LRINT_F80,
1165 RTLIB::LRINT_F128,
1166 RTLIB::LRINT_PPCF128));
1167}
1168
1169SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1170 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1171 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1172 RTLIB::LLRINT_F32,
1173 RTLIB::LLRINT_F64,
1174 RTLIB::LLRINT_F80,
1175 RTLIB::LLRINT_F128,
1176 RTLIB::LLRINT_PPCF128));
1177}
1178
1179//===----------------------------------------------------------------------===//
1180// Float Result Expansion
1181//===----------------------------------------------------------------------===//
1182
1183/// ExpandFloatResult - This method is called when the specified result of the
1184/// specified node is found to need expansion. At this point, the node may also
1185/// have invalid operands or may have other results that need promotion, we just
1186/// know that (at least) one result needs expansion.
1187void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1188 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1189 SDValue Lo, Hi;
1190 Lo = Hi = SDValue();
1191
1192 // See if the target wants to custom expand this node.
1193 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1194 return;
1195
1196 switch (N->getOpcode()) {
1197 default:
1198#ifndef NDEBUG
1199 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1200 N->dump(&DAG); dbgs() << "\n";
1201#endif
1202 llvm_unreachable("Do not know how to expand the result of this operator!");
1203
1204 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1205 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1206 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1207
1208 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1209 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1210 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1211 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1212 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1213 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1214
1215 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1216 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1218 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1220 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1221 case ISD::STRICT_FADD:
1222 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1223 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1224 case ISD::STRICT_FCEIL:
1225 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1226 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1227 case ISD::STRICT_FCOS:
1228 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1229 case ISD::STRICT_FDIV:
1230 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1231 case ISD::STRICT_FEXP:
1232 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1233 case ISD::STRICT_FEXP2:
1234 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1235 case ISD::STRICT_FFLOOR:
1236 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1237 case ISD::STRICT_FLOG:
1238 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1239 case ISD::STRICT_FLOG2:
1240 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1241 case ISD::STRICT_FLOG10:
1242 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1243 case ISD::STRICT_FMA:
1244 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1245 case ISD::STRICT_FMUL:
1246 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1248 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1249 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1251 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1252 case ISD::STRICT_FPOW:
1253 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1254 case ISD::STRICT_FPOWI:
1255 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1256 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1257 case ISD::STRICT_FRINT:
1258 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1259 case ISD::STRICT_FROUND:
1260 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1262 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1263 case ISD::STRICT_FSIN:
1264 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1265 case ISD::STRICT_FSQRT:
1266 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1267 case ISD::STRICT_FSUB:
1268 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1269 case ISD::STRICT_FTRUNC:
1270 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1271 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1274 case ISD::SINT_TO_FP:
1275 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1276 case ISD::STRICT_FREM:
1277 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1278 }
1279
1280 // If Lo/Hi is null, the sub-method took care of registering results etc.
1281 if (Lo.getNode())
1282 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1283}
1284
1285void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1286 SDValue &Hi) {
1287 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1288 assert(NVT.getSizeInBits() == 64 &&
1289 "Do not know how to expand this float constant!");
1290 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1291 SDLoc dl(N);
1293 APInt(64, C.getRawData()[1])),
1294 dl, NVT);
1296 APInt(64, C.getRawData()[0])),
1297 dl, NVT);
1298}
1299
1300void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1301 SDValue &Lo, SDValue &Hi) {
1302 bool IsStrict = N->isStrictFPOpcode();
1303 unsigned Offset = IsStrict ? 1 : 0;
1304 SDValue Op = N->getOperand(0 + Offset);
1305 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1307 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1308 Op, CallOptions, SDLoc(N),
1309 Chain);
1310 if (IsStrict)
1311 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1312 GetPairElements(Tmp.first, Lo, Hi);
1313}
1314
1315void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1316 SDValue &Lo, SDValue &Hi) {
1317 bool IsStrict = N->isStrictFPOpcode();
1318 unsigned Offset = IsStrict ? 1 : 0;
1319 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1320 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1322 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1323 Ops, CallOptions, SDLoc(N),
1324 Chain);
1325 if (IsStrict)
1326 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1327 GetPairElements(Tmp.first, Lo, Hi);
1328}
1329
1330void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1331 SDValue &Hi) {
1332 assert(N->getValueType(0) == MVT::ppcf128 &&
1333 "Logic only correct for ppcf128!");
1334 SDLoc dl(N);
1335 SDValue Tmp;
1336 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1337 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1338 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1339 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1340 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1341 ISD::SETEQ);
1342}
1343
1344void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1345 SDValue &Hi) {
1346 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1347 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1348 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1349 RTLIB::FMIN_PPCF128), Lo, Hi);
1350}
1351
1352void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1353 SDValue &Hi) {
1354 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1355 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1356 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1357 RTLIB::FMAX_PPCF128), Lo, Hi);
1358}
1359
1360void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1361 SDValue &Hi) {
1362 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1363 RTLIB::ADD_F32, RTLIB::ADD_F64,
1364 RTLIB::ADD_F80, RTLIB::ADD_F128,
1365 RTLIB::ADD_PPCF128), Lo, Hi);
1366}
1367
1368void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1369 SDValue &Hi) {
1370 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1371 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1372 RTLIB::CBRT_F128,
1373 RTLIB::CBRT_PPCF128), Lo, Hi);
1374}
1375
1376void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1377 SDValue &Lo, SDValue &Hi) {
1378 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1379 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1380 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1381 RTLIB::CEIL_PPCF128), Lo, Hi);
1382}
1383
1384void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1385 SDValue &Lo, SDValue &Hi) {
1386 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1387 RTLIB::COPYSIGN_F32,
1388 RTLIB::COPYSIGN_F64,
1389 RTLIB::COPYSIGN_F80,
1390 RTLIB::COPYSIGN_F128,
1391 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1392}
1393
1394void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1395 SDValue &Lo, SDValue &Hi) {
1396 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1397 RTLIB::COS_F32, RTLIB::COS_F64,
1398 RTLIB::COS_F80, RTLIB::COS_F128,
1399 RTLIB::COS_PPCF128), Lo, Hi);
1400}
1401
1402void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1403 SDValue &Hi) {
1404 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1405 RTLIB::DIV_F32,
1406 RTLIB::DIV_F64,
1407 RTLIB::DIV_F80,
1408 RTLIB::DIV_F128,
1409 RTLIB::DIV_PPCF128), Lo, Hi);
1410}
1411
1412void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1413 SDValue &Lo, SDValue &Hi) {
1414 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1415 RTLIB::EXP_F32, RTLIB::EXP_F64,
1416 RTLIB::EXP_F80, RTLIB::EXP_F128,
1417 RTLIB::EXP_PPCF128), Lo, Hi);
1418}
1419
1420void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1421 SDValue &Lo, SDValue &Hi) {
1422 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1423 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1424 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1425 RTLIB::EXP2_PPCF128), Lo, Hi);
1426}
1427
1428void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1429 SDValue &Lo, SDValue &Hi) {
1430 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1431 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1432 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1433 RTLIB::FLOOR_PPCF128), Lo, Hi);
1434}
1435
1436void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1437 SDValue &Lo, SDValue &Hi) {
1438 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1439 RTLIB::LOG_F32, RTLIB::LOG_F64,
1440 RTLIB::LOG_F80, RTLIB::LOG_F128,
1441 RTLIB::LOG_PPCF128), Lo, Hi);
1442}
1443
1444void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1445 SDValue &Lo, SDValue &Hi) {
1446 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1447 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1448 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1449 RTLIB::LOG2_PPCF128), Lo, Hi);
1450}
1451
1452void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1453 SDValue &Lo, SDValue &Hi) {
1454 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1455 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1456 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1457 RTLIB::LOG10_PPCF128), Lo, Hi);
1458}
1459
1460void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1461 SDValue &Hi) {
1462 bool IsStrict = N->isStrictFPOpcode();
1463 unsigned Offset = IsStrict ? 1 : 0;
1464 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1465 N->getOperand(2 + Offset) };
1466 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1468 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1469 RTLIB::FMA_F32,
1470 RTLIB::FMA_F64,
1471 RTLIB::FMA_F80,
1472 RTLIB::FMA_F128,
1473 RTLIB::FMA_PPCF128),
1474 N->getValueType(0), Ops, CallOptions,
1475 SDLoc(N), Chain);
1476 if (IsStrict)
1477 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1478 GetPairElements(Tmp.first, Lo, Hi);
1479}
1480
1481void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1482 SDValue &Hi) {
1483 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1484 RTLIB::MUL_F32,
1485 RTLIB::MUL_F64,
1486 RTLIB::MUL_F80,
1487 RTLIB::MUL_F128,
1488 RTLIB::MUL_PPCF128), Lo, Hi);
1489}
1490
1491void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1492 SDValue &Lo, SDValue &Hi) {
1493 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1494 RTLIB::NEARBYINT_F32,
1495 RTLIB::NEARBYINT_F64,
1496 RTLIB::NEARBYINT_F80,
1497 RTLIB::NEARBYINT_F128,
1498 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1499}
1500
1501void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1502 SDValue &Hi) {
1503 SDLoc dl(N);
1504 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1505 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1506 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1507}
1508
1509void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1510 SDValue &Hi) {
1511 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1512 SDLoc dl(N);
1513 bool IsStrict = N->isStrictFPOpcode();
1514
1515 SDValue Chain;
1516 if (IsStrict) {
1517 // If the expanded type is the same as the input type, just bypass the node.
1518 if (NVT == N->getOperand(1).getValueType()) {
1519 Hi = N->getOperand(1);
1520 Chain = N->getOperand(0);
1521 } else {
1522 // Other we need to extend.
1523 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1524 { N->getOperand(0), N->getOperand(1) });
1525 Chain = Hi.getValue(1);
1526 }
1527 } else {
1528 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1529 }
1530
1532 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1533
1534 if (IsStrict)
1535 ReplaceValueWith(SDValue(N, 1), Chain);
1536}
1537
1538void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1539 SDValue &Lo, SDValue &Hi) {
1540 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1541 RTLIB::POW_F32, RTLIB::POW_F64,
1542 RTLIB::POW_F80, RTLIB::POW_F128,
1543 RTLIB::POW_PPCF128), Lo, Hi);
1544}
1545
1546void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1547 SDValue &Lo, SDValue &Hi) {
1548 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1549}
1550
1551void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1552 SDValue &Lo, SDValue &Hi) {
1553 assert(N->getValueType(0) == MVT::ppcf128 &&
1554 "Logic only correct for ppcf128!");
1555
1556 SDLoc dl(N);
1557 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1558 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1559 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1560}
1561
1562void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1563 SDValue &Lo, SDValue &Hi) {
1564 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1565 RTLIB::REM_F32, RTLIB::REM_F64,
1566 RTLIB::REM_F80, RTLIB::REM_F128,
1567 RTLIB::REM_PPCF128), Lo, Hi);
1568}
1569
1570void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1571 SDValue &Lo, SDValue &Hi) {
1572 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1573 RTLIB::RINT_F32, RTLIB::RINT_F64,
1574 RTLIB::RINT_F80, RTLIB::RINT_F128,
1575 RTLIB::RINT_PPCF128), Lo, Hi);
1576}
1577
1578void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1579 SDValue &Lo, SDValue &Hi) {
1580 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1581 RTLIB::ROUND_F32,
1582 RTLIB::ROUND_F64,
1583 RTLIB::ROUND_F80,
1584 RTLIB::ROUND_F128,
1585 RTLIB::ROUND_PPCF128), Lo, Hi);
1586}
1587
1588void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1589 SDValue &Lo, SDValue &Hi) {
1590 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1591 RTLIB::ROUNDEVEN_F32,
1592 RTLIB::ROUNDEVEN_F64,
1593 RTLIB::ROUNDEVEN_F80,
1594 RTLIB::ROUNDEVEN_F128,
1595 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1596}
1597
1598void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1599 SDValue &Lo, SDValue &Hi) {
1600 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1601 RTLIB::SIN_F32, RTLIB::SIN_F64,
1602 RTLIB::SIN_F80, RTLIB::SIN_F128,
1603 RTLIB::SIN_PPCF128), Lo, Hi);
1604}
1605
1606void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1607 SDValue &Lo, SDValue &Hi) {
1608 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1609 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1610 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1611 RTLIB::SQRT_PPCF128), Lo, Hi);
1612}
1613
1614void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1615 SDValue &Hi) {
1616 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1617 RTLIB::SUB_F32,
1618 RTLIB::SUB_F64,
1619 RTLIB::SUB_F80,
1620 RTLIB::SUB_F128,
1621 RTLIB::SUB_PPCF128), Lo, Hi);
1622}
1623
1624void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1625 SDValue &Lo, SDValue &Hi) {
1626 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1627 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1628 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1629 RTLIB::TRUNC_PPCF128), Lo, Hi);
1630}
1631
1632void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1633 SDValue &Hi) {
1634 if (ISD::isNormalLoad(N)) {
1635 ExpandRes_NormalLoad(N, Lo, Hi);
1636 return;
1637 }
1638
1639 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1640 LoadSDNode *LD = cast<LoadSDNode>(N);
1641 SDValue Chain = LD->getChain();
1642 SDValue Ptr = LD->getBasePtr();
1643 SDLoc dl(N);
1644
1645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1646 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1647 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1648
1649 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1650 LD->getMemoryVT(), LD->getMemOperand());
1651
1652 // Remember the chain.
1653 Chain = Hi.getValue(1);
1654
1655 // The low part is zero.
1657 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1658
1659 // Modified the chain - switch anything that used the old chain to use the
1660 // new one.
1661 ReplaceValueWith(SDValue(LD, 1), Chain);
1662}
1663
1664void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1665 SDValue &Hi) {
1666 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1667 EVT VT = N->getValueType(0);
1668 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1669 bool Strict = N->isStrictFPOpcode();
1670 SDValue Src = N->getOperand(Strict ? 1 : 0);
1671 EVT SrcVT = Src.getValueType();
1672 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1673 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1674 SDLoc dl(N);
1675 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1676
1677 // TODO: Any other flags to propagate?
1679 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1680
1681 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1682 // When promoting partial word types to i32 we must honor the signedness,
1683 // though.
1684 if (SrcVT.bitsLE(MVT::i32)) {
1685 // The integer can be represented exactly in an f64.
1687 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1688 if (Strict) {
1689 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1690 {Chain, Src}, Flags);
1691 Chain = Hi.getValue(1);
1692 } else
1693 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1694 } else {
1695 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1696 if (SrcVT.bitsLE(MVT::i64)) {
1698 MVT::i64, Src);
1699 LC = RTLIB::SINTTOFP_I64_PPCF128;
1700 } else if (SrcVT.bitsLE(MVT::i128)) {
1701 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1702 LC = RTLIB::SINTTOFP_I128_PPCF128;
1703 }
1704 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1705
1707 CallOptions.setSExt(true);
1708 std::pair<SDValue, SDValue> Tmp =
1709 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1710 if (Strict)
1711 Chain = Tmp.second;
1712 GetPairElements(Tmp.first, Lo, Hi);
1713 }
1714
1715 // No need to complement for unsigned 32-bit integers
1716 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1717 if (Strict)
1718 ReplaceValueWith(SDValue(N, 1), Chain);
1719
1720 return;
1721 }
1722
1723 // Unsigned - fix up the SINT_TO_FP value just calculated.
1724 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1725 // keep semantics correctness if the integer is not exactly representable
1726 // here. See ExpandLegalINT_TO_FP.
1727 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1728 SrcVT = Src.getValueType();
1729
1730 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1731 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1732 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1733 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1734 ArrayRef<uint64_t> Parts;
1735
1736 switch (SrcVT.getSimpleVT().SimpleTy) {
1737 default:
1738 llvm_unreachable("Unsupported UINT_TO_FP!");
1739 case MVT::i32:
1740 Parts = TwoE32;
1741 break;
1742 case MVT::i64:
1743 Parts = TwoE64;
1744 break;
1745 case MVT::i128:
1746 Parts = TwoE128;
1747 break;
1748 }
1749
1750 // TODO: Are there other fast-math-flags to propagate to this FADD?
1751 SDValue NewLo = DAG.getConstantFP(
1752 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1753 if (Strict) {
1754 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1755 {Chain, Hi, NewLo}, Flags);
1756 Chain = Lo.getValue(1);
1757 ReplaceValueWith(SDValue(N, 1), Chain);
1758 } else
1759 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1760 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1761 Lo, Hi, ISD::SETLT);
1762 GetPairElements(Lo, Lo, Hi);
1763}
1764
1765
1766//===----------------------------------------------------------------------===//
1767// Float Operand Expansion
1768//===----------------------------------------------------------------------===//
1769
1770/// ExpandFloatOperand - This method is called when the specified operand of the
1771/// specified node is found to need expansion. At this point, all of the result
1772/// types of the node are known to be legal, but other operands of the node may
1773/// need promotion or expansion as well as the specified one.
1774bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1775 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1776 SDValue Res = SDValue();
1777
1778 // See if the target wants to custom expand this node.
1779 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1780 return false;
1781
1782 switch (N->getOpcode()) {
1783 default:
1784#ifndef NDEBUG
1785 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1786 N->dump(&DAG); dbgs() << "\n";
1787#endif
1788 llvm_unreachable("Do not know how to expand this operator's operand!");
1789
1790 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1791 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1792 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1793
1794 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1795 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1797 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1800 case ISD::FP_TO_SINT:
1801 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1802 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1803 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1804 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1805 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1806 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1807 case ISD::STRICT_FSETCC:
1809 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1810 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1811 OpNo); break;
1812 }
1813
1814 // If the result is null, the sub-method took care of registering results etc.
1815 if (!Res.getNode()) return false;
1816
1817 // If the result is N, the sub-method updated N in place. Tell the legalizer
1818 // core about this.
1819 if (Res.getNode() == N)
1820 return true;
1821
1822 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1823 "Invalid operand expansion");
1824
1825 ReplaceValueWith(SDValue(N, 0), Res);
1826 return false;
1827}
1828
1829/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1830/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1831void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1832 SDValue &NewRHS,
1833 ISD::CondCode &CCCode,
1834 const SDLoc &dl, SDValue &Chain,
1835 bool IsSignaling) {
1836 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1837 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1838 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1839
1840 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1841
1842 // FIXME: This generated code sucks. We want to generate
1843 // FCMPU crN, hi1, hi2
1844 // BNE crN, L:
1845 // FCMPU crN, lo1, lo2
1846 // The following can be improved, but not that much.
1847 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1848 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1849 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1850 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1851 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1852 RHSLo, CCCode, OutputChain, IsSignaling);
1853 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1854 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1855 Tmp1 =
1856 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1857 ISD::SETUNE, OutputChain, IsSignaling);
1858 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1859 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1860 RHSHi, CCCode, OutputChain, IsSignaling);
1861 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1862 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1863 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1864 NewRHS = SDValue(); // LHS is the result, not a compare.
1865 Chain = OutputChain;
1866}
1867
1868SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1869 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1870 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1871 SDValue Chain;
1872 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1873
1874 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1875 // against zero to select between true and false values.
1876 if (!NewRHS.getNode()) {
1877 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1878 CCCode = ISD::SETNE;
1879 }
1880
1881 // Update N to have the operands specified.
1882 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1883 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1884 N->getOperand(4)), 0);
1885}
1886
1887SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1888 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1889 "Logic only correct for ppcf128!");
1890 SDValue Lo, Hi;
1891 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1892 // The ppcf128 value is providing only the sign; take it from the
1893 // higher-order double (which must have the larger magnitude).
1894 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1895 N->getValueType(0), N->getOperand(0), Hi);
1896}
1897
1898SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1899 bool IsStrict = N->isStrictFPOpcode();
1900 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1901 "Logic only correct for ppcf128!");
1902 SDValue Lo, Hi;
1903 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1904
1905 if (!IsStrict)
1906 // Round it the rest of the way (e.g. to f32) if needed.
1907 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1908 N->getValueType(0), Hi, N->getOperand(1));
1909
1910 // Eliminate the node if the input float type is the same as the output float
1911 // type.
1912 if (Hi.getValueType() == N->getValueType(0)) {
1913 // Connect the output chain to the input chain, unlinking the node.
1914 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1915 ReplaceValueWith(SDValue(N, 0), Hi);
1916 return SDValue();
1917 }
1918
1919 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1920 {N->getValueType(0), MVT::Other},
1921 {N->getOperand(0), Hi, N->getOperand(2)});
1922 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1923 ReplaceValueWith(SDValue(N, 0), Expansion);
1924 return SDValue();
1925}
1926
1927SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1928 EVT RVT = N->getValueType(0);
1929 SDLoc dl(N);
1930
1931 bool IsStrict = N->isStrictFPOpcode();
1932 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1933 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1934 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1935 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1936
1937 EVT NVT;
1938 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1939 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1940 "Unsupported FP_TO_XINT!");
1942 std::pair<SDValue, SDValue> Tmp =
1943 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1944 if (!IsStrict)
1945 return Tmp.first;
1946
1947 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1948 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1949 return SDValue();
1950}
1951
1952SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1953 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1954 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1955 SDValue Chain;
1956 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1957
1958 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1959 // against zero to select between true and false values.
1960 if (!NewRHS.getNode()) {
1961 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1962 CCCode = ISD::SETNE;
1963 }
1964
1965 // Update N to have the operands specified.
1966 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1967 N->getOperand(2), N->getOperand(3),
1968 DAG.getCondCode(CCCode)), 0);
1969}
1970
1971SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1972 bool IsStrict = N->isStrictFPOpcode();
1973 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1974 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1975 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1976 ISD::CondCode CCCode =
1977 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1978 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
1979 N->getOpcode() == ISD::STRICT_FSETCCS);
1980
1981 // FloatExpandSetCCOperands always returned a scalar.
1982 assert(!NewRHS.getNode() && "Expect to return scalar");
1983 assert(NewLHS.getValueType() == N->getValueType(0) &&
1984 "Unexpected setcc expansion!");
1985 if (Chain) {
1986 ReplaceValueWith(SDValue(N, 0), NewLHS);
1987 ReplaceValueWith(SDValue(N, 1), Chain);
1988 return SDValue();
1989 }
1990 return NewLHS;
1991}
1992
1993SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1994 if (ISD::isNormalStore(N))
1995 return ExpandOp_NormalStore(N, OpNo);
1996
1997 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1998 assert(OpNo == 1 && "Can only expand the stored value so far");
1999 StoreSDNode *ST = cast<StoreSDNode>(N);
2000
2001 SDValue Chain = ST->getChain();
2002 SDValue Ptr = ST->getBasePtr();
2003
2004 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2005 ST->getValue().getValueType());
2006 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2007 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2008 (void)NVT;
2009
2010 SDValue Lo, Hi;
2011 GetExpandedOp(ST->getValue(), Lo, Hi);
2012
2013 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2014 ST->getMemoryVT(), ST->getMemOperand());
2015}
2016
2017SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2018 EVT RVT = N->getValueType(0);
2019 EVT RetVT = N->getOperand(0).getValueType();
2021 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2022 RTLIB::LROUND_F32,
2023 RTLIB::LROUND_F64,
2024 RTLIB::LROUND_F80,
2025 RTLIB::LROUND_F128,
2026 RTLIB::LROUND_PPCF128),
2027 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2028}
2029
2030SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2031 EVT RVT = N->getValueType(0);
2032 EVT RetVT = N->getOperand(0).getValueType();
2034 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2035 RTLIB::LLROUND_F32,
2036 RTLIB::LLROUND_F64,
2037 RTLIB::LLROUND_F80,
2038 RTLIB::LLROUND_F128,
2039 RTLIB::LLROUND_PPCF128),
2040 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2041}
2042
2043SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2044 EVT RVT = N->getValueType(0);
2045 EVT RetVT = N->getOperand(0).getValueType();
2047 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2048 RTLIB::LRINT_F32,
2049 RTLIB::LRINT_F64,
2050 RTLIB::LRINT_F80,
2051 RTLIB::LRINT_F128,
2052 RTLIB::LRINT_PPCF128),
2053 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2054}
2055
2056SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2057 EVT RVT = N->getValueType(0);
2058 EVT RetVT = N->getOperand(0).getValueType();
2060 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2061 RTLIB::LLRINT_F32,
2062 RTLIB::LLRINT_F64,
2063 RTLIB::LLRINT_F80,
2064 RTLIB::LLRINT_F128,
2065 RTLIB::LLRINT_PPCF128),
2066 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2067}
2068
2069//===----------------------------------------------------------------------===//
2070// Float Operand Promotion
2071//===----------------------------------------------------------------------===//
2072//
2073
2075 if (OpVT == MVT::f16) {
2076 return ISD::FP16_TO_FP;
2077 } else if (RetVT == MVT::f16) {
2078 return ISD::FP_TO_FP16;
2079 } else if (OpVT == MVT::bf16) {
2080 return ISD::BF16_TO_FP;
2081 } else if (RetVT == MVT::bf16) {
2082 return ISD::FP_TO_BF16;
2083 }
2084
2085 report_fatal_error("Attempt at an invalid promotion-related conversion");
2086}
2087
2088bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2089 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2090 dbgs() << "\n");
2091 SDValue R = SDValue();
2092
2093 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2094 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2095 return false;
2096 }
2097
2098 // Nodes that use a promotion-requiring floating point operand, but doesn't
2099 // produce a promotion-requiring floating point result, need to be legalized
2100 // to use the promoted float operand. Nodes that produce at least one
2101 // promotion-requiring floating point result have their operands legalized as
2102 // a part of PromoteFloatResult.
2103 switch (N->getOpcode()) {
2104 default:
2105 #ifndef NDEBUG
2106 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2107 N->dump(&DAG); dbgs() << "\n";
2108 #endif
2109 llvm_unreachable("Do not know how to promote this operator's operand!");
2110
2111 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2112 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2113 case ISD::FP_TO_SINT:
2114 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2117 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2118 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2119 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2120 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2121 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2122 }
2123
2124 if (R.getNode())
2125 ReplaceValueWith(SDValue(N, 0), R);
2126 return false;
2127}
2128
2129SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2130 SDValue Op = N->getOperand(0);
2131 EVT OpVT = Op->getValueType(0);
2132
2133 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2134 EVT PromotedVT = Promoted->getValueType(0);
2135
2136 // Convert the promoted float value to the desired IVT.
2137 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2138 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2139 IVT, Promoted);
2140 // The final result type might not be an scalar so we need a bitcast. The
2141 // bitcast will be further legalized if needed.
2142 return DAG.getBitcast(N->getValueType(0), Convert);
2143}
2144
2145// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2146// PromoteFloatRes_FCOPYSIGN.
2147SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2148 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2149 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2150
2151 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2152 N->getOperand(0), Op1);
2153}
2154
2155// Convert the promoted float value to the desired integer type
2156SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2157 SDValue Op = GetPromotedFloat(N->getOperand(0));
2158 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2159}
2160
2161SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2162 unsigned OpNo) {
2163 SDValue Op = GetPromotedFloat(N->getOperand(0));
2164 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2165 N->getOperand(1));
2166}
2167
2168SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2169 SDValue Op = GetPromotedFloat(N->getOperand(0));
2170 EVT VT = N->getValueType(0);
2171
2172 // Desired VT is same as promoted type. Use promoted float directly.
2173 if (VT == Op->getValueType(0))
2174 return Op;
2175
2176 // Else, extend the promoted float value to the desired VT.
2177 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2178}
2179
2180// Promote the float operands used for comparison. The true- and false-
2181// operands have the same type as the result and are promoted, if needed, by
2182// PromoteFloatRes_SELECT_CC
2183SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2184 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2185 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2186
2187 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2188 LHS, RHS, N->getOperand(2), N->getOperand(3),
2189 N->getOperand(4));
2190}
2191
2192// Construct a SETCC that compares the promoted values and sets the conditional
2193// code.
2194SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2195 EVT VT = N->getValueType(0);
2196 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2197 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2198 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2199
2200 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2201
2202}
2203
2204// Lower the promoted Float down to the integer value of same size and construct
2205// a STORE of the integer value.
2206SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2207 StoreSDNode *ST = cast<StoreSDNode>(N);
2208 SDValue Val = ST->getValue();
2209 SDLoc DL(N);
2210
2211 SDValue Promoted = GetPromotedFloat(Val);
2212 EVT VT = ST->getOperand(1).getValueType();
2213 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2214
2215 SDValue NewVal;
2216 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2217 IVT, Promoted);
2218
2219 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2220 ST->getMemOperand());
2221}
2222
2223//===----------------------------------------------------------------------===//
2224// Float Result Promotion
2225//===----------------------------------------------------------------------===//
2226
2227void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2228 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2229 dbgs() << "\n");
2230 SDValue R = SDValue();
2231
2232 // See if the target wants to custom expand this node.
2233 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2234 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2235 return;
2236 }
2237
2238 switch (N->getOpcode()) {
2239 // These opcodes cannot appear if promotion of FP16 is done in the backend
2240 // instead of Clang
2241 case ISD::FP16_TO_FP:
2242 case ISD::FP_TO_FP16:
2243 default:
2244#ifndef NDEBUG
2245 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2246 N->dump(&DAG); dbgs() << "\n";
2247#endif
2248 llvm_unreachable("Do not know how to promote this operator's result!");
2249
2250 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2251 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2253 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2254 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2255
2256 // Unary FP Operations
2257 case ISD::FABS:
2258 case ISD::FCBRT:
2259 case ISD::FCEIL:
2260 case ISD::FCOS:
2261 case ISD::FEXP:
2262 case ISD::FEXP2:
2263 case ISD::FFLOOR:
2264 case ISD::FLOG:
2265 case ISD::FLOG2:
2266 case ISD::FLOG10:
2267 case ISD::FNEARBYINT:
2268 case ISD::FNEG:
2269 case ISD::FRINT:
2270 case ISD::FROUND:
2271 case ISD::FROUNDEVEN:
2272 case ISD::FSIN:
2273 case ISD::FSQRT:
2274 case ISD::FTRUNC:
2275 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2276
2277 // Binary FP Operations
2278 case ISD::FADD:
2279 case ISD::FDIV:
2280 case ISD::FMAXIMUM:
2281 case ISD::FMINIMUM:
2282 case ISD::FMAXNUM:
2283 case ISD::FMINNUM:
2284 case ISD::FMUL:
2285 case ISD::FPOW:
2286 case ISD::FREM:
2287 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2288
2289 case ISD::FMA: // FMA is same as FMAD
2290 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2291
2292 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2293
2294 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2295 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2296 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2297 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2298
2299 case ISD::SINT_TO_FP:
2300 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2301 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2302 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2307 R = PromoteFloatRes_VECREDUCE(N);
2308 break;
2311 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2312 break;
2313 }
2314
2315 if (R.getNode())
2316 SetPromotedFloat(SDValue(N, ResNo), R);
2317}
2318
2319// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2320// At this point, it is not possible to determine if the bitcast value is
2321// eventually stored to memory or promoted to f32 or promoted to a floating
2322// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2323// STORE promotion handlers.
2324SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2325 EVT VT = N->getValueType(0);
2326 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2327 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2328 // bitcast will be legalized further if necessary.
2329 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2330 N->getOperand(0).getValueType().getSizeInBits());
2331 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2332 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2333}
2334
2335SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2336 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2337 EVT VT = N->getValueType(0);
2338 SDLoc DL(N);
2339
2340 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2341 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2342 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2343 IVT);
2344
2345 // Convert the Constant to the desired FP type
2346 // FIXME We might be able to do the conversion during compilation and get rid
2347 // of it from the object code
2348 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2349 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2350}
2351
2352// If the Index operand is a constant, try to redirect the extract operation to
2353// the correct legalized vector. If not, bit-convert the input vector to
2354// equivalent integer vector. Extract the element as an (bit-cast) integer
2355// value and convert it to the promoted type.
2356SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2357 SDLoc DL(N);
2358
2359 // If the index is constant, try to extract the value from the legalized
2360 // vector type.
2361 if (isa<ConstantSDNode>(N->getOperand(1))) {
2362 SDValue Vec = N->getOperand(0);
2363 SDValue Idx = N->getOperand(1);
2364 EVT VecVT = Vec->getValueType(0);
2365 EVT EltVT = VecVT.getVectorElementType();
2366
2367 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2368
2369 switch (getTypeAction(VecVT)) {
2370 default: break;
2372 SDValue Res = GetScalarizedVector(N->getOperand(0));
2373 ReplaceValueWith(SDValue(N, 0), Res);
2374 return SDValue();
2375 }
2377 Vec = GetWidenedVector(Vec);
2378 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2379 ReplaceValueWith(SDValue(N, 0), Res);
2380 return SDValue();
2381 }
2383 SDValue Lo, Hi;
2384 GetSplitVector(Vec, Lo, Hi);
2385
2386 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2387 SDValue Res;
2388 if (IdxVal < LoElts)
2389 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2390 else
2391 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2392 DAG.getConstant(IdxVal - LoElts, DL,
2393 Idx.getValueType()));
2394 ReplaceValueWith(SDValue(N, 0), Res);
2395 return SDValue();
2396 }
2397
2398 }
2399 }
2400
2401 // Bit-convert the input vector to the equivalent integer vector
2402 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2403 EVT IVT = NewOp.getValueType().getVectorElementType();
2404
2405 // Extract the element as an (bit-cast) integer value
2406 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2407 NewOp, N->getOperand(1));
2408
2409 // Convert the element to the desired FP type
2410 EVT VT = N->getValueType(0);
2411 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2412 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2413}
2414
2415// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2416// needs promotion, so does the argument X. Note that Y, if needed, will be
2417// handled during operand promotion.
2418SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2419 EVT VT = N->getValueType(0);
2420 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2421 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2422
2423 SDValue Op1 = N->getOperand(1);
2424
2425 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2426}
2427
2428// Unary operation where the result and the operand have PromoteFloat type
2429// action. Construct a new SDNode with the promoted float value of the old
2430// operand.
2431SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2432 EVT VT = N->getValueType(0);
2433 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2434 SDValue Op = GetPromotedFloat(N->getOperand(0));
2435
2436 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2437}
2438
2439// Binary operations where the result and both operands have PromoteFloat type
2440// action. Construct a new SDNode with the promoted float values of the old
2441// operands.
2442SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2443 EVT VT = N->getValueType(0);
2444 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2445 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2446 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2447 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2448}
2449
2450SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2451 EVT VT = N->getValueType(0);
2452 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2453 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2454 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2455 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2456
2457 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2458}
2459
2460// Promote the Float (first) operand and retain the Integer (second) operand
2461SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2462 EVT VT = N->getValueType(0);
2463 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2464 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2465 SDValue Op1 = N->getOperand(1);
2466
2467 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2468}
2469
2470// Explicit operation to reduce precision. Reduce the value to half precision
2471// and promote it back to the legal type.
2472SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2473 SDLoc DL(N);
2474
2475 SDValue Op = N->getOperand(0);
2476 EVT VT = N->getValueType(0);
2477 EVT OpVT = Op->getValueType(0);
2478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2479 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2480
2481 // Round promoted float to desired precision
2482 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2483 // Promote it back to the legal output type
2484 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2485}
2486
2487SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2488 LoadSDNode *L = cast<LoadSDNode>(N);
2489 EVT VT = N->getValueType(0);
2490
2491 // Load the value as an integer value with the same number of bits.
2492 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2493 SDValue newL = DAG.getLoad(
2494 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2495 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2496 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2497 // Legalize the chain result by replacing uses of the old value chain with the
2498 // new one
2499 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2500
2501 // Convert the integer value to the desired FP type
2502 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2503 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2504}
2505
2506// Construct a new SELECT node with the promoted true- and false- values.
2507SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2508 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2509 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2510
2511 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2512 N->getOperand(0), TrueVal, FalseVal);
2513}
2514
2515// Construct a new SELECT_CC node with the promoted true- and false- values.
2516// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2517SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2518 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2519 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2520
2521 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2522 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2523 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2524}
2525
2526// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2527// float type.
2528SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2529 SDLoc DL(N);
2530 EVT VT = N->getValueType(0);
2531 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2532 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2533 // Round the value to the desired precision (that of the source type).
2534 return DAG.getNode(
2535 ISD::FP_EXTEND, DL, NVT,
2536 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2537 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2538}
2539
2540SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2541 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2542 N->getValueType(0)));
2543}
2544
2545SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2546 // Expand and promote recursively.
2547 // TODO: This is non-optimal, but dealing with the concurrently happening
2548 // vector-legalization is non-trivial. We could do something similar to
2549 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2550 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2551 return SDValue();
2552}
2553
2554SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2555 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2556 return SDValue();
2557}
2558
2559SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2560 EVT VT = N->getValueType(0);
2561
2562 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2563 SDLoc SL(N);
2564
2565 SDValue CastVal = BitConvertToInteger(AM->getVal());
2566 EVT CastVT = CastVal.getValueType();
2567
2568 SDValue NewAtomic
2569 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2570 DAG.getVTList(CastVT, MVT::Other),
2571 { AM->getChain(), AM->getBasePtr(), CastVal },
2572 AM->getMemOperand());
2573
2574 SDValue Result = NewAtomic;
2575
2576 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2577 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2578 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2579 NewAtomic);
2580 }
2581
2582 // Legalize the chain result by replacing uses of the old value chain with the
2583 // new one
2584 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2585
2586 return Result;
2587
2588}
2589
2590//===----------------------------------------------------------------------===//
2591// Half Result Soft Promotion
2592//===----------------------------------------------------------------------===//
2593
2594void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2595 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2596 N->dump(&DAG); dbgs() << "\n");
2597 SDValue R = SDValue();
2598
2599 // See if the target wants to custom expand this node.
2600 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2601 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2602 return;
2603 }
2604
2605 switch (N->getOpcode()) {
2606 default:
2607#ifndef NDEBUG
2608 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2609 N->dump(&DAG); dbgs() << "\n";
2610#endif
2611 llvm_unreachable("Do not know how to soft promote this operator's result!");
2612
2613 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2614 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2616 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2617 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2619 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2620
2621 // Unary FP Operations
2622 case ISD::FABS:
2623 case ISD::FCBRT:
2624 case ISD::FCEIL:
2625 case ISD::FCOS:
2626 case ISD::FEXP:
2627 case ISD::FEXP2:
2628 case ISD::FFLOOR:
2629 case ISD::FLOG:
2630 case ISD::FLOG2:
2631 case ISD::FLOG10:
2632 case ISD::FNEARBYINT:
2633 case ISD::FNEG:
2634 case ISD::FREEZE:
2635 case ISD::FRINT:
2636 case ISD::FROUND:
2637 case ISD::FROUNDEVEN:
2638 case ISD::FSIN:
2639 case ISD::FSQRT:
2640 case ISD::FTRUNC:
2641 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2642
2643 // Binary FP Operations
2644 case ISD::FADD:
2645 case ISD::FDIV:
2646 case ISD::FMAXIMUM:
2647 case ISD::FMINIMUM:
2648 case ISD::FMAXNUM:
2649 case ISD::FMINNUM:
2650 case ISD::FMUL:
2651 case ISD::FPOW:
2652 case ISD::FREM:
2653 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2654
2655 case ISD::FMA: // FMA is same as FMAD
2656 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2657
2658 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2659
2660 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2661 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2662 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2663 case ISD::SINT_TO_FP:
2664 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2665 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2666 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2671 R = SoftPromoteHalfRes_VECREDUCE(N);
2672 break;
2675 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2676 break;
2677 }
2678
2679 if (R.getNode())
2680 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2681}
2682
2683SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2684 return BitConvertToInteger(N->getOperand(0));
2685}
2686
2687SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2688 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2689
2690 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2691 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2692 MVT::i16);
2693}
2694
2695SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2696 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2698 NewOp.getValueType().getVectorElementType(), NewOp,
2699 N->getOperand(1));
2700}
2701
2702SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2703 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2704 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2705 SDLoc dl(N);
2706
2707 EVT LVT = LHS.getValueType();
2708 EVT RVT = RHS.getValueType();
2709
2710 unsigned LSize = LVT.getSizeInBits();
2711 unsigned RSize = RVT.getSizeInBits();
2712
2713 // First get the sign bit of second operand.
2714 SDValue SignBit = DAG.getNode(
2715 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2716 DAG.getConstant(RSize - 1, dl,
2717 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2718 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2719
2720 // Shift right or sign-extend it if the two operands have different types.
2721 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2722 if (SizeDiff > 0) {
2723 SignBit =
2724 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2725 DAG.getConstant(SizeDiff, dl,
2726 TLI.getShiftAmountTy(SignBit.getValueType(),
2727 DAG.getDataLayout())));
2728 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2729 } else if (SizeDiff < 0) {
2730 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2731 SignBit =
2732 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2733 DAG.getConstant(-SizeDiff, dl,
2734 TLI.getShiftAmountTy(SignBit.getValueType(),
2735 DAG.getDataLayout())));
2736 }
2737
2738 // Clear the sign bit of the first operand.
2739 SDValue Mask = DAG.getNode(
2740 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2741 DAG.getConstant(LSize - 1, dl,
2742 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2743 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2744 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2745
2746 // Or the value with the sign bit.
2747 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2748}
2749
2750SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2751 EVT OVT = N->getValueType(0);
2752 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2753 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2754 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2755 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2756 SDLoc dl(N);
2757
2758 // Promote to the larger FP type.
2759 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2760 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2761 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2762 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2763
2764 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2765
2766 // Convert back to FP16 as an integer.
2767 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2768}
2769
2770SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2771 EVT OVT = N->getValueType(0);
2772 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2773 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2774 SDValue Op1 = N->getOperand(1);
2775 SDLoc dl(N);
2776
2777 // Promote to the larger FP type.
2778 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2779
2780 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2781
2782 // Convert back to FP16 as an integer.
2783 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2784}
2785
2786SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2787 EVT RVT = N->getValueType(0);
2788 EVT SVT = N->getOperand(0).getValueType();
2789
2790 if (N->isStrictFPOpcode()) {
2791 assert(RVT == MVT::f16);
2792 SDValue Res =
2794 {N->getOperand(0), N->getOperand(1)});
2795 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2796 return Res;
2797 }
2798
2799 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2800 N->getOperand(0));
2801}
2802
2803SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2804 LoadSDNode *L = cast<LoadSDNode>(N);
2805
2806 // Load the value as an integer value with the same number of bits.
2807 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2808 SDValue NewL =
2809 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2810 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2811 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2812 L->getMemOperand()->getFlags(), L->getAAInfo());
2813 // Legalize the chain result by replacing uses of the old value chain with the
2814 // new one
2815 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2816 return NewL;
2817}
2818
2819SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2820 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2821 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2822 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2823 Op2);
2824}
2825
2826SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2827 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2828 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2829 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2830 N->getOperand(0), N->getOperand(1), Op2, Op3,
2831 N->getOperand(4));
2832}
2833
2834SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2835 EVT OVT = N->getValueType(0);
2836 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2837 SDLoc dl(N);
2838
2839 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2840
2841 // Round the value to the softened type.
2842 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2843}
2844
2845SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2846 return DAG.getUNDEF(MVT::i16);
2847}
2848
2849SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2850 EVT OVT = N->getValueType(0);
2851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2852 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2853 SDLoc dl(N);
2854
2855 // Promote to the larger FP type.
2856 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2857
2858 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2859
2860 // Convert back to FP16 as an integer.
2861 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2862}
2863
2864SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2865 EVT OVT = N->getValueType(0);
2866 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2867 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2868 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2869 SDLoc dl(N);
2870
2871 // Promote to the larger FP type.
2872 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2873 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2874 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2875
2876 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2877
2878 // Convert back to FP16 as an integer.
2879 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2880}
2881
2882SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2883 // Expand and soften recursively.
2884 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2885 return SDValue();
2886}
2887
2888SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2889 // Expand and soften.
2890 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2891 return SDValue();
2892}
2893
2894//===----------------------------------------------------------------------===//
2895// Half Operand Soft Promotion
2896//===----------------------------------------------------------------------===//
2897
2898bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2899 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2900 N->dump(&DAG); dbgs() << "\n");
2901 SDValue Res = SDValue();
2902
2903 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2904 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2905 return false;
2906 }
2907
2908 // Nodes that use a promotion-requiring floating point operand, but doesn't
2909 // produce a soft promotion-requiring floating point result, need to be
2910 // legalized to use the soft promoted float operand. Nodes that produce at
2911 // least one soft promotion-requiring floating point result have their
2912 // operands legalized as a part of PromoteFloatResult.
2913 switch (N->getOpcode()) {
2914 default:
2915 #ifndef NDEBUG
2916 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2917 N->dump(&DAG); dbgs() << "\n";
2918 #endif
2919 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2920
2921 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2922 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2923 case ISD::FP_TO_SINT:
2924 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2927 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2929 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2930 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2931 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2932 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2933 case ISD::STACKMAP:
2934 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
2935 break;
2936 case ISD::PATCHPOINT:
2937 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
2938 break;
2939 }
2940
2941 if (!Res.getNode())
2942 return false;
2943
2944 assert(Res.getNode() != N && "Expected a new node!");
2945
2946 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2947 "Invalid operand expansion");
2948
2949 ReplaceValueWith(SDValue(N, 0), Res);
2950 return false;
2951}
2952
2953SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2954 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2955
2956 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2957}
2958
2959SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2960 unsigned OpNo) {
2961 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2962 SDValue Op1 = N->getOperand(1);
2963 EVT RVT = Op1.getValueType();
2964 SDLoc dl(N);
2965
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2967
2968 Op1 = GetSoftPromotedHalf(Op1);
2969 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
2970
2971 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2972 Op1);
2973}
2974
2975SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2976 EVT RVT = N->getValueType(0);
2977 bool IsStrict = N->isStrictFPOpcode();
2978 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2979 EVT SVT = Op.getValueType();
2980 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
2981
2982 if (IsStrict) {
2983 assert(SVT == MVT::f16);
2984 SDValue Res =
2986 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
2987 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2988 ReplaceValueWith(SDValue(N, 0), Res);
2989 return SDValue();
2990 }
2991
2992 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
2993}
2994
2995SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
2996 EVT RVT = N->getValueType(0);
2997 SDValue Op = N->getOperand(0);
2998 EVT SVT = Op.getValueType();
2999 SDLoc dl(N);
3000
3001 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3002
3003 Op = GetSoftPromotedHalf(Op);
3004
3005 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3006
3007 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3008}
3009
3010SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3011 EVT RVT = N->getValueType(0);
3012 SDValue Op = N->getOperand(0);
3013 EVT SVT = Op.getValueType();
3014 SDLoc dl(N);
3015
3016 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3017
3018 Op = GetSoftPromotedHalf(Op);
3019
3020 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3021
3022 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3023 N->getOperand(1));
3024}
3025
3026SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3027 unsigned OpNo) {
3028 assert(OpNo == 0 && "Can only soften the comparison values");
3029 SDValue Op0 = N->getOperand(0);
3030 SDValue Op1 = N->getOperand(1);
3031 SDLoc dl(N);
3032
3033 EVT SVT = Op0.getValueType();
3034 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3035
3036 Op0 = GetSoftPromotedHalf(Op0);
3037 Op1 = GetSoftPromotedHalf(Op1);
3038
3039 // Promote to the larger FP type.
3040 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3041 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3042 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3043
3044 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3045 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3046}
3047
3048SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3049 SDValue Op0 = N->getOperand(0);
3050 SDValue Op1 = N->getOperand(1);
3051 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3052 SDLoc dl(N);
3053
3054 EVT SVT = Op0.getValueType();
3055 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3056
3057 Op0 = GetSoftPromotedHalf(Op0);
3058 Op1 = GetSoftPromotedHalf(Op1);
3059
3060 // Promote to the larger FP type.
3061 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3062 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3063 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3064
3065 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3066}
3067
3068SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3069 assert(OpNo == 1 && "Can only soften the stored value!");
3070 StoreSDNode *ST = cast<StoreSDNode>(N);
3071 SDValue Val = ST->getValue();
3072 SDLoc dl(N);
3073
3074 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3075 SDValue Promoted = GetSoftPromotedHalf(Val);
3076 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3077 ST->getMemOperand());
3078}
3079
3080SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3081 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3082 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3083 SDValue Op = N->getOperand(OpNo);
3084 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3086 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3087
3088 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3089 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3090
3091 return SDValue(); // Signal that we replaced the node ourselves.
3092}
3093
3094SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3095 unsigned OpNo) {
3096 assert(OpNo >= 7);
3097 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3098 SDValue Op = N->getOperand(OpNo);
3099 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3101 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3102
3103 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3104 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3105
3106 return SDValue(); // Signal that we replaced the node ourselves.
3107}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
@ Flags
Definition: TextStubV5.cpp:93
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1174
Class for arbitrary precision integers.
Definition: APInt.h:75
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:209
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:557
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:42
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ LAST_INTEGER_VALUETYPE
@ FIRST_INTEGER_VALUETYPE
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1276
@ ConstantFP
Definition: ISDOpcodes.h:77
@ STRICT_FCEIL
Definition: ISDOpcodes.h:425
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:486
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:482
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1292
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:908
@ STRICT_FLOG2
Definition: ISDOpcodes.h:420
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1289
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1293
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:923
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1020
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:499
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:726
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:429
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1162
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:911
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:910
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:423
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:424
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:741
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:955
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:883
@ STRICT_FROUND
Definition: ISDOpcodes.h:427
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:426
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:428
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:917
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:442
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:464
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:441
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:968
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:469
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1290
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:419
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
@ STRICT_FEXP2
Definition: ISDOpcodes.h:417
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1199
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:865
@ STRICT_LLROUND
Definition: ISDOpcodes.h:431
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:422
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:851
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1089
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1277
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
@ STRICT_FRINT
Definition: ISDOpcodes.h:421
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:62
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
#define N
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:251
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:226
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)