LLVM  12.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "legalize-types"
27 
28 /// GetFPLibCall - Return the right libcall for the given floating point type.
30  RTLIB::Libcall Call_F32,
31  RTLIB::Libcall Call_F64,
32  RTLIB::Libcall Call_F80,
33  RTLIB::Libcall Call_F128,
34  RTLIB::Libcall Call_PPCF128) {
35  return
36  VT == MVT::f32 ? Call_F32 :
37  VT == MVT::f64 ? Call_F64 :
38  VT == MVT::f80 ? Call_F80 :
39  VT == MVT::f128 ? Call_F128 :
40  VT == MVT::ppcf128 ? Call_PPCF128 :
41  RTLIB::UNKNOWN_LIBCALL;
42 }
43 
44 //===----------------------------------------------------------------------===//
45 // Convert Float Results to Integer
46 //===----------------------------------------------------------------------===//
47 
48 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
49  LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
50  dbgs() << "\n");
51  SDValue R = SDValue();
52 
53  switch (N->getOpcode()) {
54  default:
55 #ifndef NDEBUG
56  dbgs() << "SoftenFloatResult #" << ResNo << ": ";
57  N->dump(&DAG); dbgs() << "\n";
58 #endif
59  llvm_unreachable("Do not know how to soften the result of this operator!");
60 
61  case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
62  case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
63  case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
64  case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
66  R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
67  case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
69  case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
71  case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
72  case ISD::STRICT_FADD:
73  case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
74  case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
75  case ISD::STRICT_FCEIL:
76  case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
77  case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
78  case ISD::STRICT_FCOS:
79  case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
80  case ISD::STRICT_FDIV:
81  case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
82  case ISD::STRICT_FEXP:
83  case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
84  case ISD::STRICT_FEXP2:
85  case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
86  case ISD::STRICT_FFLOOR:
87  case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
88  case ISD::STRICT_FLOG:
89  case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
90  case ISD::STRICT_FLOG2:
91  case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
92  case ISD::STRICT_FLOG10:
93  case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
94  case ISD::STRICT_FMA:
95  case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
96  case ISD::STRICT_FMUL:
97  case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
99  case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
100  case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
102  case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
104  case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
105  case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
106  case ISD::STRICT_FPOW:
107  case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
108  case ISD::STRICT_FPOWI:
109  case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
110  case ISD::STRICT_FREM:
111  case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
112  case ISD::STRICT_FRINT:
113  case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
114  case ISD::STRICT_FROUND:
115  case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
117  case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
118  case ISD::STRICT_FSIN:
119  case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
120  case ISD::STRICT_FSQRT:
121  case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
122  case ISD::STRICT_FSUB:
123  case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
124  case ISD::STRICT_FTRUNC:
125  case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
126  case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
127  case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
128  case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
129  case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
130  case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
133  case ISD::SINT_TO_FP:
134  case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
135  case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
136  case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
137  }
138 
139  // If R is null, the sub-method took care of registering the result.
140  if (R.getNode()) {
141  assert(R.getNode() != N);
142  SetSoftenedFloat(SDValue(N, ResNo), R);
143  }
144 }
145 
146 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
147  bool IsStrict = N->isStrictFPOpcode();
148  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
149  unsigned Offset = IsStrict ? 1 : 0;
150  assert(N->getNumOperands() == (1 + Offset) &&
151  "Unexpected number of operands!");
152  SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
153  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
155  EVT OpVT = N->getOperand(0 + Offset).getValueType();
156  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
157  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
158  CallOptions, SDLoc(N),
159  Chain);
160  if (IsStrict)
161  ReplaceValueWith(SDValue(N, 1), Tmp.second);
162  return Tmp.first;
163 }
164 
165 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
166  bool IsStrict = N->isStrictFPOpcode();
167  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
168  unsigned Offset = IsStrict ? 1 : 0;
169  assert(N->getNumOperands() == (2 + Offset) &&
170  "Unexpected number of operands!");
171  SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
172  GetSoftenedFloat(N->getOperand(1 + Offset)) };
173  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
175  EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
176  N->getOperand(1 + Offset).getValueType() };
177  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
178  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
179  CallOptions, SDLoc(N),
180  Chain);
181  if (IsStrict)
182  ReplaceValueWith(SDValue(N, 1), Tmp.second);
183  return Tmp.first;
184 }
185 
186 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
187  return BitConvertToInteger(N->getOperand(0));
188 }
189 
190 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
191  EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
192  return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
193  GetSoftenedFloat(N->getOperand(0)));
194 }
195 
196 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
197  unsigned ResNo) {
198  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
199  return BitConvertToInteger(Op);
200 }
201 
202 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
203  // Convert the inputs to integers, and build a new pair out of them.
204  return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
205  TLI.getTypeToTransformTo(*DAG.getContext(),
206  N->getValueType(0)),
207  BitConvertToInteger(N->getOperand(0)),
208  BitConvertToInteger(N->getOperand(1)));
209 }
210 
211 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
212  ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
213  // In ppcf128, the high 64 bits are always first in memory regardless
214  // of Endianness. LLVM's APFloat representation is not Endian sensitive,
215  // and so always converts into a 128-bit APInt in a non-Endian-sensitive
216  // way. However, APInt's are serialized in an Endian-sensitive fashion,
217  // so on big-Endian targets, the two doubles are output in the wrong
218  // order. Fix this by manually flipping the order of the high 64 bits
219  // and the low 64 bits here.
220  if (DAG.getDataLayout().isBigEndian() &&
222  uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
223  CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
224  APInt Val(128, words);
225  return DAG.getConstant(Val, SDLoc(CN),
226  TLI.getTypeToTransformTo(*DAG.getContext(),
227  CN->getValueType(0)));
228  } else {
229  return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
230  TLI.getTypeToTransformTo(*DAG.getContext(),
231  CN->getValueType(0)));
232  }
233 }
234 
235 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
236  SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
237  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
239  NewOp, N->getOperand(1));
240 }
241 
242 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
243  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
244  unsigned Size = NVT.getSizeInBits();
245 
246  // Mask = ~(1 << (Size-1))
247  APInt API = APInt::getAllOnesValue(Size);
248  API.clearBit(Size - 1);
249  SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
250  SDValue Op = GetSoftenedFloat(N->getOperand(0));
251  return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
252 }
253 
254 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
255  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
256  RTLIB::FMIN_F32,
257  RTLIB::FMIN_F64,
258  RTLIB::FMIN_F80,
259  RTLIB::FMIN_F128,
260  RTLIB::FMIN_PPCF128));
261 }
262 
263 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
264  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
265  RTLIB::FMAX_F32,
266  RTLIB::FMAX_F64,
267  RTLIB::FMAX_F80,
268  RTLIB::FMAX_F128,
269  RTLIB::FMAX_PPCF128));
270 }
271 
272 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
273  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
274  RTLIB::ADD_F32,
275  RTLIB::ADD_F64,
276  RTLIB::ADD_F80,
277  RTLIB::ADD_F128,
278  RTLIB::ADD_PPCF128));
279 }
280 
281 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
282  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
283  RTLIB::CBRT_F32,
284  RTLIB::CBRT_F64,
285  RTLIB::CBRT_F80,
286  RTLIB::CBRT_F128,
287  RTLIB::CBRT_PPCF128));
288 }
289 
290 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
291  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
292  RTLIB::CEIL_F32,
293  RTLIB::CEIL_F64,
294  RTLIB::CEIL_F80,
295  RTLIB::CEIL_F128,
296  RTLIB::CEIL_PPCF128));
297 }
298 
299 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
300  SDValue LHS = GetSoftenedFloat(N->getOperand(0));
301  SDValue RHS = BitConvertToInteger(N->getOperand(1));
302  SDLoc dl(N);
303 
304  EVT LVT = LHS.getValueType();
305  EVT RVT = RHS.getValueType();
306 
307  unsigned LSize = LVT.getSizeInBits();
308  unsigned RSize = RVT.getSizeInBits();
309 
310  // First get the sign bit of second operand.
311  SDValue SignBit = DAG.getNode(
312  ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
313  DAG.getConstant(RSize - 1, dl,
314  TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
315  SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
316 
317  // Shift right or sign-extend it if the two operands have different types.
318  int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
319  if (SizeDiff > 0) {
320  SignBit =
321  DAG.getNode(ISD::SRL, dl, RVT, SignBit,
322  DAG.getConstant(SizeDiff, dl,
323  TLI.getShiftAmountTy(SignBit.getValueType(),
324  DAG.getDataLayout())));
325  SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
326  } else if (SizeDiff < 0) {
327  SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
328  SignBit =
329  DAG.getNode(ISD::SHL, dl, LVT, SignBit,
330  DAG.getConstant(-SizeDiff, dl,
331  TLI.getShiftAmountTy(SignBit.getValueType(),
332  DAG.getDataLayout())));
333  }
334 
335  // Clear the sign bit of the first operand.
336  SDValue Mask = DAG.getNode(
337  ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
338  DAG.getConstant(LSize - 1, dl,
339  TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
340  Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
341  LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
342 
343  // Or the value with the sign bit.
344  return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
345 }
346 
347 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
348  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
349  RTLIB::COS_F32,
350  RTLIB::COS_F64,
351  RTLIB::COS_F80,
352  RTLIB::COS_F128,
353  RTLIB::COS_PPCF128));
354 }
355 
356 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
357  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
358  RTLIB::DIV_F32,
359  RTLIB::DIV_F64,
360  RTLIB::DIV_F80,
361  RTLIB::DIV_F128,
362  RTLIB::DIV_PPCF128));
363 }
364 
365 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
366  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
367  RTLIB::EXP_F32,
368  RTLIB::EXP_F64,
369  RTLIB::EXP_F80,
370  RTLIB::EXP_F128,
371  RTLIB::EXP_PPCF128));
372 }
373 
374 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
375  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
376  RTLIB::EXP2_F32,
377  RTLIB::EXP2_F64,
378  RTLIB::EXP2_F80,
379  RTLIB::EXP2_F128,
380  RTLIB::EXP2_PPCF128));
381 }
382 
383 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
384  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
385  RTLIB::FLOOR_F32,
386  RTLIB::FLOOR_F64,
387  RTLIB::FLOOR_F80,
388  RTLIB::FLOOR_F128,
389  RTLIB::FLOOR_PPCF128));
390 }
391 
392 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
393  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
394  RTLIB::LOG_F32,
395  RTLIB::LOG_F64,
396  RTLIB::LOG_F80,
397  RTLIB::LOG_F128,
398  RTLIB::LOG_PPCF128));
399 }
400 
401 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
402  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
403  RTLIB::LOG2_F32,
404  RTLIB::LOG2_F64,
405  RTLIB::LOG2_F80,
406  RTLIB::LOG2_F128,
407  RTLIB::LOG2_PPCF128));
408 }
409 
410 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
411  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
412  RTLIB::LOG10_F32,
413  RTLIB::LOG10_F64,
414  RTLIB::LOG10_F80,
415  RTLIB::LOG10_F128,
416  RTLIB::LOG10_PPCF128));
417 }
418 
419 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
420  bool IsStrict = N->isStrictFPOpcode();
421  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
422  unsigned Offset = IsStrict ? 1 : 0;
423  SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
424  GetSoftenedFloat(N->getOperand(1 + Offset)),
425  GetSoftenedFloat(N->getOperand(2 + Offset)) };
426  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
428  EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
429  N->getOperand(1 + Offset).getValueType(),
430  N->getOperand(2 + Offset).getValueType() };
431  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
432  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
434  RTLIB::FMA_F32,
435  RTLIB::FMA_F64,
436  RTLIB::FMA_F80,
437  RTLIB::FMA_F128,
438  RTLIB::FMA_PPCF128),
439  NVT, Ops, CallOptions, SDLoc(N), Chain);
440  if (IsStrict)
441  ReplaceValueWith(SDValue(N, 1), Tmp.second);
442  return Tmp.first;
443 }
444 
445 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
446  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
447  RTLIB::MUL_F32,
448  RTLIB::MUL_F64,
449  RTLIB::MUL_F80,
450  RTLIB::MUL_F128,
451  RTLIB::MUL_PPCF128));
452 }
453 
454 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
455  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
456  RTLIB::NEARBYINT_F32,
457  RTLIB::NEARBYINT_F64,
458  RTLIB::NEARBYINT_F80,
459  RTLIB::NEARBYINT_F128,
460  RTLIB::NEARBYINT_PPCF128));
461 }
462 
463 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
464  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
465  SDLoc dl(N);
466 
467  // Expand Y = FNEG(X) -> Y = X ^ sign mask
468  APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
469  return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
470  DAG.getConstant(SignMask, dl, NVT));
471 }
472 
473 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
474  bool IsStrict = N->isStrictFPOpcode();
475  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
476  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
477 
478  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
479 
480  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
481  Op = GetPromotedFloat(Op);
482  // If the promotion did the FP_EXTEND to the destination type for us,
483  // there's nothing left to do here.
484  if (Op.getValueType() == N->getValueType(0))
485  return BitConvertToInteger(Op);
486  }
487 
488  // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's
489  // entirely possible for both f16 and f32 to be legal, so use the fully
490  // hard-float FP_EXTEND rather than FP16_TO_FP.
491  if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) {
492  if (IsStrict) {
493  Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
494  { MVT::f32, MVT::Other }, { Chain, Op });
495  Chain = Op.getValue(1);
496  } else {
497  Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
498  }
499  }
500 
502  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
504  EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
505  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
506  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
507  CallOptions, SDLoc(N),
508  Chain);
509  if (IsStrict)
510  ReplaceValueWith(SDValue(N, 1), Tmp.second);
511  return Tmp.first;
512 }
513 
514 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
515 // nodes?
516 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
517  EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
518  SDValue Op = N->getOperand(0);
520  EVT OpsVT[1] = { N->getOperand(0).getValueType() };
521  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
522  SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
523  CallOptions, SDLoc(N)).first;
524  if (N->getValueType(0) == MVT::f32)
525  return Res32;
526 
527  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
529  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
530  return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
531 }
532 
533 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
534  bool IsStrict = N->isStrictFPOpcode();
535  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
536  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
537  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
539  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
541  EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
542  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
543  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
544  CallOptions, SDLoc(N),
545  Chain);
546  if (IsStrict)
547  ReplaceValueWith(SDValue(N, 1), Tmp.second);
548  return Tmp.first;
549 }
550 
551 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
552  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
553  RTLIB::POW_F32,
554  RTLIB::POW_F64,
555  RTLIB::POW_F80,
556  RTLIB::POW_F128,
557  RTLIB::POW_PPCF128));
558 }
559 
560 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
561  bool IsStrict = N->isStrictFPOpcode();
562  unsigned Offset = IsStrict ? 1 : 0;
563  assert(N->getOperand(1 + Offset).getValueType() == MVT::i32 &&
564  "Unsupported power type!");
566  RTLIB::POWI_F32,
567  RTLIB::POWI_F64,
568  RTLIB::POWI_F80,
569  RTLIB::POWI_F128,
570  RTLIB::POWI_PPCF128);
571  if (!TLI.getLibcallName(LC)) {
572  // Some targets don't have a powi libcall; use pow instead.
573  // FIXME: Implement this if some target needs it.
574  DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
575  return DAG.getUNDEF(N->getValueType(0));
576  }
577 
578  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
579  SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
580  N->getOperand(1 + Offset) };
581  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
583  EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
584  N->getOperand(1 + Offset).getValueType() };
585  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
586  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
587  CallOptions, SDLoc(N),
588  Chain);
589  if (IsStrict)
590  ReplaceValueWith(SDValue(N, 1), Tmp.second);
591  return Tmp.first;
592 }
593 
594 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
595  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
596  RTLIB::REM_F32,
597  RTLIB::REM_F64,
598  RTLIB::REM_F80,
599  RTLIB::REM_F128,
600  RTLIB::REM_PPCF128));
601 }
602 
603 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
604  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
605  RTLIB::RINT_F32,
606  RTLIB::RINT_F64,
607  RTLIB::RINT_F80,
608  RTLIB::RINT_F128,
609  RTLIB::RINT_PPCF128));
610 }
611 
612 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
613  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
614  RTLIB::ROUND_F32,
615  RTLIB::ROUND_F64,
616  RTLIB::ROUND_F80,
617  RTLIB::ROUND_F128,
618  RTLIB::ROUND_PPCF128));
619 }
620 
621 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
622  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
623  RTLIB::ROUNDEVEN_F32,
624  RTLIB::ROUNDEVEN_F64,
625  RTLIB::ROUNDEVEN_F80,
626  RTLIB::ROUNDEVEN_F128,
627  RTLIB::ROUNDEVEN_PPCF128));
628 }
629 
630 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
631  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
632  RTLIB::SIN_F32,
633  RTLIB::SIN_F64,
634  RTLIB::SIN_F80,
635  RTLIB::SIN_F128,
636  RTLIB::SIN_PPCF128));
637 }
638 
639 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
640  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
641  RTLIB::SQRT_F32,
642  RTLIB::SQRT_F64,
643  RTLIB::SQRT_F80,
644  RTLIB::SQRT_F128,
645  RTLIB::SQRT_PPCF128));
646 }
647 
648 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
649  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
650  RTLIB::SUB_F32,
651  RTLIB::SUB_F64,
652  RTLIB::SUB_F80,
653  RTLIB::SUB_F128,
654  RTLIB::SUB_PPCF128));
655 }
656 
657 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
658  return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
659  RTLIB::TRUNC_F32,
660  RTLIB::TRUNC_F64,
661  RTLIB::TRUNC_F80,
662  RTLIB::TRUNC_F128,
663  RTLIB::TRUNC_PPCF128));
664 }
665 
666 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
667  LoadSDNode *L = cast<LoadSDNode>(N);
668  EVT VT = N->getValueType(0);
669  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
670  SDLoc dl(N);
671 
672  auto MMOFlags =
673  L->getMemOperand()->getFlags() &
675  SDValue NewL;
676  if (L->getExtensionType() == ISD::NON_EXTLOAD) {
677  NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
678  L->getChain(), L->getBasePtr(), L->getOffset(),
679  L->getPointerInfo(), NVT, L->getOriginalAlign(),
680  MMOFlags, L->getAAInfo());
681  // Legalized the chain result - switch anything that used the old chain to
682  // use the new one.
683  ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
684  return NewL;
685  }
686 
687  // Do a non-extending load followed by FP_EXTEND.
688  NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
689  dl, L->getChain(), L->getBasePtr(), L->getOffset(),
690  L->getPointerInfo(), L->getMemoryVT(),
691  L->getOriginalAlign(), MMOFlags, L->getAAInfo());
692  // Legalized the chain result - switch anything that used the old chain to
693  // use the new one.
694  ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
695  auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
696  return BitConvertToInteger(ExtendNode);
697 }
698 
699 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
700  SDValue LHS = GetSoftenedFloat(N->getOperand(1));
701  SDValue RHS = GetSoftenedFloat(N->getOperand(2));
702  return DAG.getSelect(SDLoc(N),
703  LHS.getValueType(), N->getOperand(0), LHS, RHS);
704 }
705 
706 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
707  SDValue LHS = GetSoftenedFloat(N->getOperand(2));
708  SDValue RHS = GetSoftenedFloat(N->getOperand(3));
709  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
710  LHS.getValueType(), N->getOperand(0),
711  N->getOperand(1), LHS, RHS, N->getOperand(4));
712 }
713 
714 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
715  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
716  N->getValueType(0)));
717 }
718 
719 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
720  SDValue Chain = N->getOperand(0); // Get the chain.
721  SDValue Ptr = N->getOperand(1); // Get the pointer.
722  EVT VT = N->getValueType(0);
723  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
724  SDLoc dl(N);
725 
726  SDValue NewVAARG;
727  NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
728  N->getConstantOperandVal(3));
729 
730  // Legalized the chain result - switch anything that used the old chain to
731  // use the new one.
732  if (N != NewVAARG.getValue(1).getNode())
733  ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
734  return NewVAARG;
735 }
736 
737 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
738  bool IsStrict = N->isStrictFPOpcode();
739  bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
741  EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
742  EVT RVT = N->getValueType(0);
743  EVT NVT = EVT();
744  SDLoc dl(N);
745 
746  // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
747  // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
748  // match. Look for an appropriate libcall.
749  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
750  for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
751  t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
752  NVT = (MVT::SimpleValueType)t;
753  // The source needs to big enough to hold the operand.
754  if (NVT.bitsGE(SVT))
755  LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
756  }
757  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
758 
759  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
760  // Sign/zero extend the argument if the libcall takes a larger type.
761  SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
762  NVT, N->getOperand(IsStrict ? 1 : 0));
764  CallOptions.setSExt(Signed);
765  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
766  std::pair<SDValue, SDValue> Tmp =
767  TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
768  Op, CallOptions, dl, Chain);
769 
770  if (IsStrict)
771  ReplaceValueWith(SDValue(N, 1), Tmp.second);
772  return Tmp.first;
773 }
774 
775 
776 //===----------------------------------------------------------------------===//
777 // Convert Float Operand to Integer
778 //===----------------------------------------------------------------------===//
779 
780 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
781  LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
782  dbgs() << "\n");
783  SDValue Res = SDValue();
784 
785  switch (N->getOpcode()) {
786  default:
787 #ifndef NDEBUG
788  dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
789  N->dump(&DAG); dbgs() << "\n";
790 #endif
791  llvm_unreachable("Do not know how to soften this operator's operand!");
792 
793  case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
794  case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
795  case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
797  case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
800  case ISD::FP_TO_SINT:
801  case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
802  case ISD::STRICT_LROUND:
803  case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
804  case ISD::STRICT_LLROUND:
805  case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
806  case ISD::STRICT_LRINT:
807  case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
808  case ISD::STRICT_LLRINT:
809  case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
810  case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
811  case ISD::STRICT_FSETCC:
812  case ISD::STRICT_FSETCCS:
813  case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
814  case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
815  case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
816  }
817 
818  // If the result is null, the sub-method took care of registering results etc.
819  if (!Res.getNode()) return false;
820 
821  // If the result is N, the sub-method updated N in place. Tell the legalizer
822  // core about this to re-analyze.
823  if (Res.getNode() == N)
824  return true;
825 
826  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
827  "Invalid operand softening");
828 
829  ReplaceValueWith(SDValue(N, 0), Res);
830  return false;
831 }
832 
833 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
834  SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
835 
836  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
837 }
838 
839 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
840  // We actually deal with the partially-softened FP_TO_FP16 node too, which
841  // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
844 
845  bool IsStrict = N->isStrictFPOpcode();
846  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
847  EVT SVT = Op.getValueType();
848  EVT RVT = N->getValueType(0);
849  EVT FloatRVT = N->getOpcode() == ISD::FP_TO_FP16 ? MVT::f16 : RVT;
850 
851  RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
852  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
853 
854  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
855  Op = GetSoftenedFloat(Op);
857  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
858  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
859  CallOptions, SDLoc(N),
860  Chain);
861  if (IsStrict) {
862  ReplaceValueWith(SDValue(N, 1), Tmp.second);
863  ReplaceValueWith(SDValue(N, 0), Tmp.first);
864  return SDValue();
865  }
866  return Tmp.first;
867 }
868 
869 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
870  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
871  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
872 
873  EVT VT = NewLHS.getValueType();
874  NewLHS = GetSoftenedFloat(NewLHS);
875  NewRHS = GetSoftenedFloat(NewRHS);
876  TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
877  N->getOperand(2), N->getOperand(3));
878 
879  // If softenSetCCOperands returned a scalar, we need to compare the result
880  // against zero to select between true and false values.
881  if (!NewRHS.getNode()) {
882  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
883  CCCode = ISD::SETNE;
884  }
885 
886  // Update N to have the operands specified.
887  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
888  DAG.getCondCode(CCCode), NewLHS, NewRHS,
889  N->getOperand(4)),
890  0);
891 }
892 
893 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
894  bool IsStrict = N->isStrictFPOpcode();
895  bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
897 
898  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
899  EVT SVT = Op.getValueType();
900  EVT RVT = N->getValueType(0);
901  EVT NVT = EVT();
902  SDLoc dl(N);
903 
904  // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
905  // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
906  // match, eg. we don't have fp -> i8 conversions.
907  // Look for an appropriate libcall.
908  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
909  for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
910  IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
911  ++IntVT) {
912  NVT = (MVT::SimpleValueType)IntVT;
913  // The type needs to big enough to hold the result.
914  if (NVT.bitsGE(RVT))
915  LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) : RTLIB::getFPTOUINT(SVT, NVT);
916  }
917  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_XINT!");
918 
919  Op = GetSoftenedFloat(Op);
920  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
922  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
923  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
924  CallOptions, dl, Chain);
925 
926  // Truncate the result if the libcall returns a larger type.
927  SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
928 
929  if (!IsStrict)
930  return Res;
931 
932  ReplaceValueWith(SDValue(N, 1), Tmp.second);
933  ReplaceValueWith(SDValue(N, 0), Res);
934  return SDValue();
935 }
936 
937 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
938  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
939  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
940 
941  EVT VT = NewLHS.getValueType();
942  NewLHS = GetSoftenedFloat(NewLHS);
943  NewRHS = GetSoftenedFloat(NewRHS);
944  TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
945  N->getOperand(0), N->getOperand(1));
946 
947  // If softenSetCCOperands returned a scalar, we need to compare the result
948  // against zero to select between true and false values.
949  if (!NewRHS.getNode()) {
950  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
951  CCCode = ISD::SETNE;
952  }
953 
954  // Update N to have the operands specified.
955  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
956  N->getOperand(2), N->getOperand(3),
957  DAG.getCondCode(CCCode)),
958  0);
959 }
960 
961 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
962  bool IsStrict = N->isStrictFPOpcode();
963  SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
964  SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
965  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
966  ISD::CondCode CCCode =
967  cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
968 
969  EVT VT = Op0.getValueType();
970  SDValue NewLHS = GetSoftenedFloat(Op0);
971  SDValue NewRHS = GetSoftenedFloat(Op1);
972  TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
973  Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
974 
975  // Update N to have the operands specified.
976  if (NewRHS.getNode()) {
977  if (IsStrict)
978  NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
979  NewRHS, DAG.getCondCode(CCCode));
980  else
981  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
982  DAG.getCondCode(CCCode)), 0);
983  }
984 
985  // Otherwise, softenSetCCOperands returned a scalar, use it.
986  assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
987  "Unexpected setcc expansion!");
988 
989  if (IsStrict) {
990  ReplaceValueWith(SDValue(N, 0), NewLHS);
991  ReplaceValueWith(SDValue(N, 1), Chain);
992  return SDValue();
993  }
994  return NewLHS;
995 }
996 
997 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
998  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
999  assert(OpNo == 1 && "Can only soften the stored value!");
1000  StoreSDNode *ST = cast<StoreSDNode>(N);
1001  SDValue Val = ST->getValue();
1002  SDLoc dl(N);
1003 
1004  if (ST->isTruncatingStore())
1005  // Do an FP_ROUND followed by a non-truncating store.
1006  Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
1007  Val, DAG.getIntPtrConstant(0, dl)));
1008  else
1009  Val = GetSoftenedFloat(Val);
1010 
1011  return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1012  ST->getMemOperand());
1013 }
1014 
1015 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1016  SDValue LHS = N->getOperand(0);
1017  SDValue RHS = BitConvertToInteger(N->getOperand(1));
1018  SDLoc dl(N);
1019 
1020  EVT LVT = LHS.getValueType();
1021  EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1022  EVT RVT = RHS.getValueType();
1023 
1024  unsigned LSize = LVT.getSizeInBits();
1025  unsigned RSize = RVT.getSizeInBits();
1026 
1027  // Shift right or sign-extend it if the two operands have different types.
1028  int SizeDiff = RSize - LSize;
1029  if (SizeDiff > 0) {
1030  RHS =
1031  DAG.getNode(ISD::SRL, dl, RVT, RHS,
1032  DAG.getConstant(SizeDiff, dl,
1033  TLI.getShiftAmountTy(RHS.getValueType(),
1034  DAG.getDataLayout())));
1035  RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1036  } else if (SizeDiff < 0) {
1037  RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1038  RHS =
1039  DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1040  DAG.getConstant(-SizeDiff, dl,
1041  TLI.getShiftAmountTy(RHS.getValueType(),
1042  DAG.getDataLayout())));
1043  }
1044 
1045  RHS = DAG.getBitcast(LVT, RHS);
1046  return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1047 }
1048 
1049 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1050  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1051  bool IsStrict = N->isStrictFPOpcode();
1052  unsigned Offset = IsStrict ? 1 : 0;
1053  SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1054  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1056  EVT OpVT = N->getOperand(0 + Offset).getValueType();
1057  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1058  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1059  CallOptions, SDLoc(N),
1060  Chain);
1061  if (IsStrict) {
1062  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1063  ReplaceValueWith(SDValue(N, 0), Tmp.first);
1064  return SDValue();
1065  }
1066 
1067  return Tmp.first;
1068 }
1069 
1070 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1071  EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1072  return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1073  RTLIB::LROUND_F32,
1074  RTLIB::LROUND_F64,
1075  RTLIB::LROUND_F80,
1076  RTLIB::LROUND_F128,
1077  RTLIB::LROUND_PPCF128));
1078 }
1079 
1080 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1081  EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1082  return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1083  RTLIB::LLROUND_F32,
1084  RTLIB::LLROUND_F64,
1085  RTLIB::LLROUND_F80,
1086  RTLIB::LLROUND_F128,
1087  RTLIB::LLROUND_PPCF128));
1088 }
1089 
1090 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1091  EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1092  return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1093  RTLIB::LRINT_F32,
1094  RTLIB::LRINT_F64,
1095  RTLIB::LRINT_F80,
1096  RTLIB::LRINT_F128,
1097  RTLIB::LRINT_PPCF128));
1098 }
1099 
1100 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1101  EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1102  return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1103  RTLIB::LLRINT_F32,
1104  RTLIB::LLRINT_F64,
1105  RTLIB::LLRINT_F80,
1106  RTLIB::LLRINT_F128,
1107  RTLIB::LLRINT_PPCF128));
1108 }
1109 
1110 //===----------------------------------------------------------------------===//
1111 // Float Result Expansion
1112 //===----------------------------------------------------------------------===//
1113 
1114 /// ExpandFloatResult - This method is called when the specified result of the
1115 /// specified node is found to need expansion. At this point, the node may also
1116 /// have invalid operands or may have other results that need promotion, we just
1117 /// know that (at least) one result needs expansion.
1118 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1119  LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1120  SDValue Lo, Hi;
1121  Lo = Hi = SDValue();
1122 
1123  // See if the target wants to custom expand this node.
1124  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1125  return;
1126 
1127  switch (N->getOpcode()) {
1128  default:
1129 #ifndef NDEBUG
1130  dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1131  N->dump(&DAG); dbgs() << "\n";
1132 #endif
1133  llvm_unreachable("Do not know how to expand the result of this operator!");
1134 
1135  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1136  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1137  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1138 
1139  case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1140  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1141  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1142  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1143  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1144  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1145 
1146  case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1147  case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1148  case ISD::STRICT_FMINNUM:
1149  case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1150  case ISD::STRICT_FMAXNUM:
1151  case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1152  case ISD::STRICT_FADD:
1153  case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1154  case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1155  case ISD::STRICT_FCEIL:
1156  case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1157  case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1158  case ISD::STRICT_FCOS:
1159  case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1160  case ISD::STRICT_FDIV:
1161  case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1162  case ISD::STRICT_FEXP:
1163  case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1164  case ISD::STRICT_FEXP2:
1165  case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1166  case ISD::STRICT_FFLOOR:
1167  case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1168  case ISD::STRICT_FLOG:
1169  case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1170  case ISD::STRICT_FLOG2:
1171  case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1172  case ISD::STRICT_FLOG10:
1173  case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1174  case ISD::STRICT_FMA:
1175  case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1176  case ISD::STRICT_FMUL:
1177  case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1179  case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1180  case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1181  case ISD::STRICT_FP_EXTEND:
1182  case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1183  case ISD::STRICT_FPOW:
1184  case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1185  case ISD::STRICT_FPOWI:
1186  case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1187  case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1188  case ISD::STRICT_FRINT:
1189  case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1190  case ISD::STRICT_FROUND:
1191  case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1193  case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1194  case ISD::STRICT_FSIN:
1195  case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1196  case ISD::STRICT_FSQRT:
1197  case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1198  case ISD::STRICT_FSUB:
1199  case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1200  case ISD::STRICT_FTRUNC:
1201  case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1202  case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1203  case ISD::SINT_TO_FP:
1204  case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1205  case ISD::STRICT_FREM:
1206  case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1207  }
1208 
1209  // If Lo/Hi is null, the sub-method took care of registering results etc.
1210  if (Lo.getNode())
1211  SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1212 }
1213 
1214 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1215  SDValue &Hi) {
1216  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1217  assert(NVT.getSizeInBits() == 64 &&
1218  "Do not know how to expand this float constant!");
1219  APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1220  SDLoc dl(N);
1221  Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1222  APInt(64, C.getRawData()[1])),
1223  dl, NVT);
1224  Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1225  APInt(64, C.getRawData()[0])),
1226  dl, NVT);
1227 }
1228 
1229 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1230  SDValue &Lo, SDValue &Hi) {
1231  bool IsStrict = N->isStrictFPOpcode();
1232  unsigned Offset = IsStrict ? 1 : 0;
1233  SDValue Op = N->getOperand(0 + Offset);
1234  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1236  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1237  Op, CallOptions, SDLoc(N),
1238  Chain);
1239  if (IsStrict)
1240  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1241  GetPairElements(Tmp.first, Lo, Hi);
1242 }
1243 
1244 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1245  SDValue &Lo, SDValue &Hi) {
1246  bool IsStrict = N->isStrictFPOpcode();
1247  unsigned Offset = IsStrict ? 1 : 0;
1248  SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1249  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1251  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1252  Ops, CallOptions, SDLoc(N),
1253  Chain);
1254  if (IsStrict)
1255  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1256  GetPairElements(Tmp.first, Lo, Hi);
1257 }
1258 
1259 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1260  SDValue &Hi) {
1261  assert(N->getValueType(0) == MVT::ppcf128 &&
1262  "Logic only correct for ppcf128!");
1263  SDLoc dl(N);
1264  SDValue Tmp;
1265  GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1266  Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1267  // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1268  Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1269  DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1270  ISD::SETEQ);
1271 }
1272 
1273 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1274  SDValue &Hi) {
1275  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1276  RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1277  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1278  RTLIB::FMIN_PPCF128), Lo, Hi);
1279 }
1280 
1281 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1282  SDValue &Hi) {
1283  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1284  RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1285  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1286  RTLIB::FMAX_PPCF128), Lo, Hi);
1287 }
1288 
1289 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1290  SDValue &Hi) {
1291  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1292  RTLIB::ADD_F32, RTLIB::ADD_F64,
1293  RTLIB::ADD_F80, RTLIB::ADD_F128,
1294  RTLIB::ADD_PPCF128), Lo, Hi);
1295 }
1296 
1297 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1298  SDValue &Hi) {
1299  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1300  RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1301  RTLIB::CBRT_F128,
1302  RTLIB::CBRT_PPCF128), Lo, Hi);
1303 }
1304 
1305 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1306  SDValue &Lo, SDValue &Hi) {
1307  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1308  RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1309  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1310  RTLIB::CEIL_PPCF128), Lo, Hi);
1311 }
1312 
1313 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1314  SDValue &Lo, SDValue &Hi) {
1315  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1316  RTLIB::COPYSIGN_F32,
1317  RTLIB::COPYSIGN_F64,
1318  RTLIB::COPYSIGN_F80,
1319  RTLIB::COPYSIGN_F128,
1320  RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1321 }
1322 
1323 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1324  SDValue &Lo, SDValue &Hi) {
1325  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1326  RTLIB::COS_F32, RTLIB::COS_F64,
1327  RTLIB::COS_F80, RTLIB::COS_F128,
1328  RTLIB::COS_PPCF128), Lo, Hi);
1329 }
1330 
1331 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1332  SDValue &Hi) {
1333  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1334  RTLIB::DIV_F32,
1335  RTLIB::DIV_F64,
1336  RTLIB::DIV_F80,
1337  RTLIB::DIV_F128,
1338  RTLIB::DIV_PPCF128), Lo, Hi);
1339 }
1340 
1341 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1342  SDValue &Lo, SDValue &Hi) {
1343  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1344  RTLIB::EXP_F32, RTLIB::EXP_F64,
1345  RTLIB::EXP_F80, RTLIB::EXP_F128,
1346  RTLIB::EXP_PPCF128), Lo, Hi);
1347 }
1348 
1349 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1350  SDValue &Lo, SDValue &Hi) {
1351  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1352  RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1353  RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1354  RTLIB::EXP2_PPCF128), Lo, Hi);
1355 }
1356 
1357 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1358  SDValue &Lo, SDValue &Hi) {
1359  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1360  RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1361  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1362  RTLIB::FLOOR_PPCF128), Lo, Hi);
1363 }
1364 
1365 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1366  SDValue &Lo, SDValue &Hi) {
1367  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1368  RTLIB::LOG_F32, RTLIB::LOG_F64,
1369  RTLIB::LOG_F80, RTLIB::LOG_F128,
1370  RTLIB::LOG_PPCF128), Lo, Hi);
1371 }
1372 
1373 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1374  SDValue &Lo, SDValue &Hi) {
1375  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1376  RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1377  RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1378  RTLIB::LOG2_PPCF128), Lo, Hi);
1379 }
1380 
1381 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1382  SDValue &Lo, SDValue &Hi) {
1383  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1384  RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1385  RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1386  RTLIB::LOG10_PPCF128), Lo, Hi);
1387 }
1388 
1389 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1390  SDValue &Hi) {
1391  bool IsStrict = N->isStrictFPOpcode();
1392  unsigned Offset = IsStrict ? 1 : 0;
1393  SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1394  N->getOperand(2 + Offset) };
1395  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1397  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1398  RTLIB::FMA_F32,
1399  RTLIB::FMA_F64,
1400  RTLIB::FMA_F80,
1401  RTLIB::FMA_F128,
1402  RTLIB::FMA_PPCF128),
1403  N->getValueType(0), Ops, CallOptions,
1404  SDLoc(N), Chain);
1405  if (IsStrict)
1406  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1407  GetPairElements(Tmp.first, Lo, Hi);
1408 }
1409 
1410 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1411  SDValue &Hi) {
1412  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1413  RTLIB::MUL_F32,
1414  RTLIB::MUL_F64,
1415  RTLIB::MUL_F80,
1416  RTLIB::MUL_F128,
1417  RTLIB::MUL_PPCF128), Lo, Hi);
1418 }
1419 
1420 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1421  SDValue &Lo, SDValue &Hi) {
1422  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1423  RTLIB::NEARBYINT_F32,
1424  RTLIB::NEARBYINT_F64,
1425  RTLIB::NEARBYINT_F80,
1426  RTLIB::NEARBYINT_F128,
1427  RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1428 }
1429 
1430 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1431  SDValue &Hi) {
1432  SDLoc dl(N);
1433  GetExpandedFloat(N->getOperand(0), Lo, Hi);
1434  Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1435  Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1436 }
1437 
1438 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1439  SDValue &Hi) {
1440  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1441  SDLoc dl(N);
1442  bool IsStrict = N->isStrictFPOpcode();
1443 
1444  SDValue Chain;
1445  if (IsStrict) {
1446  // If the expanded type is the same as the input type, just bypass the node.
1447  if (NVT == N->getOperand(1).getValueType()) {
1448  Hi = N->getOperand(1);
1449  Chain = N->getOperand(0);
1450  } else {
1451  // Other we need to extend.
1452  Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1453  { N->getOperand(0), N->getOperand(1) });
1454  Chain = Hi.getValue(1);
1455  }
1456  } else {
1457  Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1458  }
1459 
1460  Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1461  APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1462 
1463  if (IsStrict)
1464  ReplaceValueWith(SDValue(N, 1), Chain);
1465 }
1466 
1467 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1468  SDValue &Lo, SDValue &Hi) {
1469  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1470  RTLIB::POW_F32, RTLIB::POW_F64,
1471  RTLIB::POW_F80, RTLIB::POW_F128,
1472  RTLIB::POW_PPCF128), Lo, Hi);
1473 }
1474 
1475 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1476  SDValue &Lo, SDValue &Hi) {
1477  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1478  RTLIB::POWI_F32, RTLIB::POWI_F64,
1479  RTLIB::POWI_F80, RTLIB::POWI_F128,
1480  RTLIB::POWI_PPCF128), Lo, Hi);
1481 }
1482 
1483 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1484  SDValue &Lo, SDValue &Hi) {
1485  assert(N->getValueType(0) == MVT::ppcf128 &&
1486  "Logic only correct for ppcf128!");
1487 
1488  SDLoc dl(N);
1489  GetExpandedFloat(N->getOperand(0), Lo, Hi);
1490  Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1491  Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1492 }
1493 
1494 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1495  SDValue &Lo, SDValue &Hi) {
1496  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1497  RTLIB::REM_F32, RTLIB::REM_F64,
1498  RTLIB::REM_F80, RTLIB::REM_F128,
1499  RTLIB::REM_PPCF128), Lo, Hi);
1500 }
1501 
1502 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1503  SDValue &Lo, SDValue &Hi) {
1504  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1505  RTLIB::RINT_F32, RTLIB::RINT_F64,
1506  RTLIB::RINT_F80, RTLIB::RINT_F128,
1507  RTLIB::RINT_PPCF128), Lo, Hi);
1508 }
1509 
1510 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1511  SDValue &Lo, SDValue &Hi) {
1512  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1513  RTLIB::ROUND_F32,
1514  RTLIB::ROUND_F64,
1515  RTLIB::ROUND_F80,
1516  RTLIB::ROUND_F128,
1517  RTLIB::ROUND_PPCF128), Lo, Hi);
1518 }
1519 
1520 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1521  SDValue &Lo, SDValue &Hi) {
1522  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1523  RTLIB::ROUNDEVEN_F32,
1524  RTLIB::ROUNDEVEN_F64,
1525  RTLIB::ROUNDEVEN_F80,
1526  RTLIB::ROUNDEVEN_F128,
1527  RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1528 }
1529 
1530 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1531  SDValue &Lo, SDValue &Hi) {
1532  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1533  RTLIB::SIN_F32, RTLIB::SIN_F64,
1534  RTLIB::SIN_F80, RTLIB::SIN_F128,
1535  RTLIB::SIN_PPCF128), Lo, Hi);
1536 }
1537 
1538 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1539  SDValue &Lo, SDValue &Hi) {
1540  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1541  RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1542  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1543  RTLIB::SQRT_PPCF128), Lo, Hi);
1544 }
1545 
1546 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1547  SDValue &Hi) {
1548  ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1549  RTLIB::SUB_F32,
1550  RTLIB::SUB_F64,
1551  RTLIB::SUB_F80,
1552  RTLIB::SUB_F128,
1553  RTLIB::SUB_PPCF128), Lo, Hi);
1554 }
1555 
1556 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1557  SDValue &Lo, SDValue &Hi) {
1558  ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559  RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1560  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1561  RTLIB::TRUNC_PPCF128), Lo, Hi);
1562 }
1563 
1564 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1565  SDValue &Hi) {
1566  if (ISD::isNormalLoad(N)) {
1567  ExpandRes_NormalLoad(N, Lo, Hi);
1568  return;
1569  }
1570 
1571  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1572  LoadSDNode *LD = cast<LoadSDNode>(N);
1573  SDValue Chain = LD->getChain();
1574  SDValue Ptr = LD->getBasePtr();
1575  SDLoc dl(N);
1576 
1577  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1578  assert(NVT.isByteSized() && "Expanded type not byte sized!");
1579  assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1580 
1581  Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1582  LD->getMemoryVT(), LD->getMemOperand());
1583 
1584  // Remember the chain.
1585  Chain = Hi.getValue(1);
1586 
1587  // The low part is zero.
1588  Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1589  APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1590 
1591  // Modified the chain - switch anything that used the old chain to use the
1592  // new one.
1593  ReplaceValueWith(SDValue(LD, 1), Chain);
1594 }
1595 
1596 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1597  SDValue &Hi) {
1598  assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1599  EVT VT = N->getValueType(0);
1600  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1601  SDValue Src = N->getOperand(0);
1602  EVT SrcVT = Src.getValueType();
1603  bool isSigned = N->getOpcode() == ISD::SINT_TO_FP;
1604  SDLoc dl(N);
1605 
1606  // First do an SINT_TO_FP, whether the original was signed or unsigned.
1607  // When promoting partial word types to i32 we must honor the signedness,
1608  // though.
1609  if (SrcVT.bitsLE(MVT::i32)) {
1610  // The integer can be represented exactly in an f64.
1611  Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1612  MVT::i32, Src);
1613  Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1614  APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1615  Hi = DAG.getNode(ISD::SINT_TO_FP, dl, NVT, Src);
1616  } else {
1617  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1618  if (SrcVT.bitsLE(MVT::i64)) {
1619  Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1620  MVT::i64, Src);
1621  LC = RTLIB::SINTTOFP_I64_PPCF128;
1622  } else if (SrcVT.bitsLE(MVT::i128)) {
1623  Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1624  LC = RTLIB::SINTTOFP_I128_PPCF128;
1625  }
1626  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1627 
1629  CallOptions.setSExt(true);
1630  Hi = TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl).first;
1631  GetPairElements(Hi, Lo, Hi);
1632  }
1633 
1634  if (isSigned)
1635  return;
1636 
1637  // Unsigned - fix up the SINT_TO_FP value just calculated.
1638  Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1639  SrcVT = Src.getValueType();
1640 
1641  // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1642  static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1643  static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1644  static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1645  ArrayRef<uint64_t> Parts;
1646 
1647  switch (SrcVT.getSimpleVT().SimpleTy) {
1648  default:
1649  llvm_unreachable("Unsupported UINT_TO_FP!");
1650  case MVT::i32:
1651  Parts = TwoE32;
1652  break;
1653  case MVT::i64:
1654  Parts = TwoE64;
1655  break;
1656  case MVT::i128:
1657  Parts = TwoE128;
1658  break;
1659  }
1660 
1661  // TODO: Are there fast-math-flags to propagate to this FADD?
1662  Lo = DAG.getNode(ISD::FADD, dl, VT, Hi,
1664  APInt(128, Parts)),
1665  dl, MVT::ppcf128));
1666  Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1667  Lo, Hi, ISD::SETLT);
1668  GetPairElements(Lo, Lo, Hi);
1669 }
1670 
1671 
1672 //===----------------------------------------------------------------------===//
1673 // Float Operand Expansion
1674 //===----------------------------------------------------------------------===//
1675 
1676 /// ExpandFloatOperand - This method is called when the specified operand of the
1677 /// specified node is found to need expansion. At this point, all of the result
1678 /// types of the node are known to be legal, but other operands of the node may
1679 /// need promotion or expansion as well as the specified one.
1680 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1681  LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1682  SDValue Res = SDValue();
1683 
1684  // See if the target wants to custom expand this node.
1685  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1686  return false;
1687 
1688  switch (N->getOpcode()) {
1689  default:
1690 #ifndef NDEBUG
1691  dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1692  N->dump(&DAG); dbgs() << "\n";
1693 #endif
1694  llvm_unreachable("Do not know how to expand this operator's operand!");
1695 
1696  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1697  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1698  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1699 
1700  case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1701  case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1702  case ISD::STRICT_FP_ROUND:
1703  case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1705  case ISD::FP_TO_SINT: Res = ExpandFloatOp_FP_TO_SINT(N); break;
1707  case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_UINT(N); break;
1708  case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1709  case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1710  case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1711  case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1712  case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1713  case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1714  case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1715  OpNo); break;
1716  }
1717 
1718  // If the result is null, the sub-method took care of registering results etc.
1719  if (!Res.getNode()) return false;
1720 
1721  // If the result is N, the sub-method updated N in place. Tell the legalizer
1722  // core about this.
1723  if (Res.getNode() == N)
1724  return true;
1725 
1726  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1727  "Invalid operand expansion");
1728 
1729  ReplaceValueWith(SDValue(N, 0), Res);
1730  return false;
1731 }
1732 
1733 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1734 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1735 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1736  SDValue &NewRHS,
1737  ISD::CondCode &CCCode,
1738  const SDLoc &dl) {
1739  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1740  GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1741  GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1742 
1743  assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1744 
1745  // FIXME: This generated code sucks. We want to generate
1746  // FCMPU crN, hi1, hi2
1747  // BNE crN, L:
1748  // FCMPU crN, lo1, lo2
1749  // The following can be improved, but not that much.
1750  SDValue Tmp1, Tmp2, Tmp3;
1751  Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1752  LHSHi, RHSHi, ISD::SETOEQ);
1753  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()),
1754  LHSLo, RHSLo, CCCode);
1755  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1756  Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1757  LHSHi, RHSHi, ISD::SETUNE);
1758  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1759  LHSHi, RHSHi, CCCode);
1760  Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1761  NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1762  NewRHS = SDValue(); // LHS is the result, not a compare.
1763 }
1764 
1765 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1766  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1767  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1768  FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
1769 
1770  // If ExpandSetCCOperands returned a scalar, we need to compare the result
1771  // against zero to select between true and false values.
1772  if (!NewRHS.getNode()) {
1773  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1774  CCCode = ISD::SETNE;
1775  }
1776 
1777  // Update N to have the operands specified.
1778  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1779  DAG.getCondCode(CCCode), NewLHS, NewRHS,
1780  N->getOperand(4)), 0);
1781 }
1782 
1783 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1785  "Logic only correct for ppcf128!");
1786  SDValue Lo, Hi;
1787  GetExpandedFloat(N->getOperand(1), Lo, Hi);
1788  // The ppcf128 value is providing only the sign; take it from the
1789  // higher-order double (which must have the larger magnitude).
1790  return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1791  N->getValueType(0), N->getOperand(0), Hi);
1792 }
1793 
1794 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1795  bool IsStrict = N->isStrictFPOpcode();
1796  assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1797  "Logic only correct for ppcf128!");
1798  SDValue Lo, Hi;
1799  GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1800 
1801  if (!IsStrict)
1802  // Round it the rest of the way (e.g. to f32) if needed.
1803  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1804  N->getValueType(0), Hi, N->getOperand(1));
1805 
1806  // Eliminate the node if the input float type is the same as the output float
1807  // type.
1808  if (Hi.getValueType() == N->getValueType(0)) {
1809  // Connect the output chain to the input chain, unlinking the node.
1810  ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1811  ReplaceValueWith(SDValue(N, 0), Hi);
1812  return SDValue();
1813  }
1814 
1815  SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1816  {N->getValueType(0), MVT::Other},
1817  {N->getOperand(0), Hi, N->getOperand(2)});
1818  ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1819  ReplaceValueWith(SDValue(N, 0), Expansion);
1820  return SDValue();
1821 }
1822 
1823 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_SINT(SDNode *N) {
1824  EVT RVT = N->getValueType(0);
1825  SDLoc dl(N);
1826 
1827  bool IsStrict = N->isStrictFPOpcode();
1828  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1829  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1831  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_SINT!");
1833  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1834  CallOptions, dl, Chain);
1835  if (!IsStrict)
1836  return Tmp.first;
1837 
1838  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1839  ReplaceValueWith(SDValue(N, 0), Tmp.first);
1840  return SDValue();
1841 }
1842 
1843 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) {
1844  EVT RVT = N->getValueType(0);
1845  SDLoc dl(N);
1846 
1847  bool IsStrict = N->isStrictFPOpcode();
1848  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1849  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1851  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_UINT!");
1853  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1854  CallOptions, dl, Chain);
1855  if (!IsStrict)
1856  return Tmp.first;
1857 
1858  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1859  ReplaceValueWith(SDValue(N, 0), Tmp.first);
1860  return SDValue();
1861 }
1862 
1863 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1864  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1865  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1866  FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
1867 
1868  // If ExpandSetCCOperands returned a scalar, we need to compare the result
1869  // against zero to select between true and false values.
1870  if (!NewRHS.getNode()) {
1871  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1872  CCCode = ISD::SETNE;
1873  }
1874 
1875  // Update N to have the operands specified.
1876  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1877  N->getOperand(2), N->getOperand(3),
1878  DAG.getCondCode(CCCode)), 0);
1879 }
1880 
1881 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1882  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1883  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
1884  FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
1885 
1886  // If ExpandSetCCOperands returned a scalar, use it.
1887  if (!NewRHS.getNode()) {
1888  assert(NewLHS.getValueType() == N->getValueType(0) &&
1889  "Unexpected setcc expansion!");
1890  return NewLHS;
1891  }
1892 
1893  // Otherwise, update N to have the operands specified.
1894  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1895  DAG.getCondCode(CCCode)), 0);
1896 }
1897 
1898 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1899  if (ISD::isNormalStore(N))
1900  return ExpandOp_NormalStore(N, OpNo);
1901 
1902  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1903  assert(OpNo == 1 && "Can only expand the stored value so far");
1904  StoreSDNode *ST = cast<StoreSDNode>(N);
1905 
1906  SDValue Chain = ST->getChain();
1907  SDValue Ptr = ST->getBasePtr();
1908 
1909  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
1910  ST->getValue().getValueType());
1911  assert(NVT.isByteSized() && "Expanded type not byte sized!");
1912  assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1913  (void)NVT;
1914 
1915  SDValue Lo, Hi;
1916  GetExpandedOp(ST->getValue(), Lo, Hi);
1917 
1918  return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
1919  ST->getMemoryVT(), ST->getMemOperand());
1920 }
1921 
1922 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
1923  EVT RVT = N->getValueType(0);
1924  EVT RetVT = N->getOperand(0).getValueType();
1926  return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1927  RTLIB::LROUND_F32,
1928  RTLIB::LROUND_F64,
1929  RTLIB::LROUND_F80,
1930  RTLIB::LROUND_F128,
1931  RTLIB::LROUND_PPCF128),
1932  RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
1933 }
1934 
1935 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
1936  EVT RVT = N->getValueType(0);
1937  EVT RetVT = N->getOperand(0).getValueType();
1939  return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1940  RTLIB::LLROUND_F32,
1941  RTLIB::LLROUND_F64,
1942  RTLIB::LLROUND_F80,
1943  RTLIB::LLROUND_F128,
1944  RTLIB::LLROUND_PPCF128),
1945  RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
1946 }
1947 
1948 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
1949  EVT RVT = N->getValueType(0);
1950  EVT RetVT = N->getOperand(0).getValueType();
1952  return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1953  RTLIB::LRINT_F32,
1954  RTLIB::LRINT_F64,
1955  RTLIB::LRINT_F80,
1956  RTLIB::LRINT_F128,
1957  RTLIB::LRINT_PPCF128),
1958  RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
1959 }
1960 
1961 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
1962  EVT RVT = N->getValueType(0);
1963  EVT RetVT = N->getOperand(0).getValueType();
1965  return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1966  RTLIB::LLRINT_F32,
1967  RTLIB::LLRINT_F64,
1968  RTLIB::LLRINT_F80,
1969  RTLIB::LLRINT_F128,
1970  RTLIB::LLRINT_PPCF128),
1971  RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
1972 }
1973 
1974 //===----------------------------------------------------------------------===//
1975 // Float Operand Promotion
1976 //===----------------------------------------------------------------------===//
1977 //
1978 
1980  if (OpVT == MVT::f16) {
1981  return ISD::FP16_TO_FP;
1982  } else if (RetVT == MVT::f16) {
1983  return ISD::FP_TO_FP16;
1984  }
1985 
1986  report_fatal_error("Attempt at an invalid promotion-related conversion");
1987 }
1988 
1989 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
1990  LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
1991  dbgs() << "\n");
1992  SDValue R = SDValue();
1993 
1994  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1995  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1996  return false;
1997  }
1998 
1999  // Nodes that use a promotion-requiring floating point operand, but doesn't
2000  // produce a promotion-requiring floating point result, need to be legalized
2001  // to use the promoted float operand. Nodes that produce at least one
2002  // promotion-requiring floating point result have their operands legalized as
2003  // a part of PromoteFloatResult.
2004  switch (N->getOpcode()) {
2005  default:
2006  #ifndef NDEBUG
2007  dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2008  N->dump(&DAG); dbgs() << "\n";
2009  #endif
2010  llvm_unreachable("Do not know how to promote this operator's operand!");
2011 
2012  case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2013  case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2014  case ISD::FP_TO_SINT:
2015  case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2016  case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2017  case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2018  case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2019  case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2020  }
2021 
2022  if (R.getNode())
2023  ReplaceValueWith(SDValue(N, 0), R);
2024  return false;
2025 }
2026 
2027 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2028  SDValue Op = N->getOperand(0);
2029  EVT OpVT = Op->getValueType(0);
2030 
2031  SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2032  EVT PromotedVT = Promoted->getValueType(0);
2033 
2034  // Convert the promoted float value to the desired IVT.
2035  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2036  SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2037  IVT, Promoted);
2038  // The final result type might not be an scalar so we need a bitcast. The
2039  // bitcast will be further legalized if needed.
2040  return DAG.getBitcast(N->getValueType(0), Convert);
2041 }
2042 
2043 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2044 // PromoteFloatRes_FCOPYSIGN.
2045 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2046  assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2047  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2048 
2049  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2050  N->getOperand(0), Op1);
2051 }
2052 
2053 // Convert the promoted float value to the desired integer type
2054 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2055  SDValue Op = GetPromotedFloat(N->getOperand(0));
2056  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2057 }
2058 
2059 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2060  SDValue Op = GetPromotedFloat(N->getOperand(0));
2061  EVT VT = N->getValueType(0);
2062 
2063  // Desired VT is same as promoted type. Use promoted float directly.
2064  if (VT == Op->getValueType(0))
2065  return Op;
2066 
2067  // Else, extend the promoted float value to the desired VT.
2068  return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2069 }
2070 
2071 // Promote the float operands used for comparison. The true- and false-
2072 // operands have the same type as the result and are promoted, if needed, by
2073 // PromoteFloatRes_SELECT_CC
2074 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2075  SDValue LHS = GetPromotedFloat(N->getOperand(0));
2076  SDValue RHS = GetPromotedFloat(N->getOperand(1));
2077 
2078  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2079  LHS, RHS, N->getOperand(2), N->getOperand(3),
2080  N->getOperand(4));
2081 }
2082 
2083 // Construct a SETCC that compares the promoted values and sets the conditional
2084 // code.
2085 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2086  EVT VT = N->getValueType(0);
2087  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2088  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2089  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2090 
2091  return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2092 
2093 }
2094 
2095 // Lower the promoted Float down to the integer value of same size and construct
2096 // a STORE of the integer value.
2097 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2098  StoreSDNode *ST = cast<StoreSDNode>(N);
2099  SDValue Val = ST->getValue();
2100  SDLoc DL(N);
2101 
2102  SDValue Promoted = GetPromotedFloat(Val);
2103  EVT VT = ST->getOperand(1).getValueType();
2104  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2105 
2106  SDValue NewVal;
2107  NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2108  IVT, Promoted);
2109 
2110  return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2111  ST->getMemOperand());
2112 }
2113 
2114 //===----------------------------------------------------------------------===//
2115 // Float Result Promotion
2116 //===----------------------------------------------------------------------===//
2117 
2118 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2119  LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2120  dbgs() << "\n");
2121  SDValue R = SDValue();
2122 
2123  // See if the target wants to custom expand this node.
2124  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2125  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2126  return;
2127  }
2128 
2129  switch (N->getOpcode()) {
2130  // These opcodes cannot appear if promotion of FP16 is done in the backend
2131  // instead of Clang
2132  case ISD::FP16_TO_FP:
2133  case ISD::FP_TO_FP16:
2134  default:
2135 #ifndef NDEBUG
2136  dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2137  N->dump(&DAG); dbgs() << "\n";
2138 #endif
2139  llvm_unreachable("Do not know how to promote this operator's result!");
2140 
2141  case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2142  case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2144  R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2145  case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2146 
2147  // Unary FP Operations
2148  case ISD::FABS:
2149  case ISD::FCBRT:
2150  case ISD::FCEIL:
2151  case ISD::FCOS:
2152  case ISD::FEXP:
2153  case ISD::FEXP2:
2154  case ISD::FFLOOR:
2155  case ISD::FLOG:
2156  case ISD::FLOG2:
2157  case ISD::FLOG10:
2158  case ISD::FNEARBYINT:
2159  case ISD::FNEG:
2160  case ISD::FRINT:
2161  case ISD::FROUND:
2162  case ISD::FROUNDEVEN:
2163  case ISD::FSIN:
2164  case ISD::FSQRT:
2165  case ISD::FTRUNC:
2166  case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2167 
2168  // Binary FP Operations
2169  case ISD::FADD:
2170  case ISD::FDIV:
2171  case ISD::FMAXIMUM:
2172  case ISD::FMINIMUM:
2173  case ISD::FMAXNUM:
2174  case ISD::FMINNUM:
2175  case ISD::FMUL:
2176  case ISD::FPOW:
2177  case ISD::FREM:
2178  case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2179 
2180  case ISD::FMA: // FMA is same as FMAD
2181  case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2182 
2183  case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2184 
2185  case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2186  case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2187  case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2188  case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2189 
2190  case ISD::SINT_TO_FP:
2191  case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2192  case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2193  case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2194  }
2195 
2196  if (R.getNode())
2197  SetPromotedFloat(SDValue(N, ResNo), R);
2198 }
2199 
2200 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2201 // At this point, it is not possible to determine if the bitcast value is
2202 // eventually stored to memory or promoted to f32 or promoted to a floating
2203 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2204 // STORE promotion handlers.
2205 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2206  EVT VT = N->getValueType(0);
2207  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2208  // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2209  // bitcast will be legalized further if necessary.
2210  EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2212  SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2213  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2214 }
2215 
2216 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2217  ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2218  EVT VT = N->getValueType(0);
2219  SDLoc DL(N);
2220 
2221  // Get the (bit-cast) APInt of the APFloat and build an integer constant
2222  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2223  SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2224  IVT);
2225 
2226  // Convert the Constant to the desired FP type
2227  // FIXME We might be able to do the conversion during compilation and get rid
2228  // of it from the object code
2229  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2230  return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2231 }
2232 
2233 // If the Index operand is a constant, try to redirect the extract operation to
2234 // the correct legalized vector. If not, bit-convert the input vector to
2235 // equivalent integer vector. Extract the element as an (bit-cast) integer
2236 // value and convert it to the promoted type.
2237 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2238  SDLoc DL(N);
2239 
2240  // If the index is constant, try to extract the value from the legalized
2241  // vector type.
2242  if (isa<ConstantSDNode>(N->getOperand(1))) {
2243  SDValue Vec = N->getOperand(0);
2244  SDValue Idx = N->getOperand(1);
2245  EVT VecVT = Vec->getValueType(0);
2246  EVT EltVT = VecVT.getVectorElementType();
2247 
2248  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2249 
2250  switch (getTypeAction(VecVT)) {
2251  default: break;
2253  SDValue Res = GetScalarizedVector(N->getOperand(0));
2254  ReplaceValueWith(SDValue(N, 0), Res);
2255  return SDValue();
2256  }
2258  Vec = GetWidenedVector(Vec);
2259  SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2260  ReplaceValueWith(SDValue(N, 0), Res);
2261  return SDValue();
2262  }
2264  SDValue Lo, Hi;
2265  GetSplitVector(Vec, Lo, Hi);
2266 
2267  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2268  SDValue Res;
2269  if (IdxVal < LoElts)
2270  Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2271  else
2272  Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2273  DAG.getConstant(IdxVal - LoElts, DL,
2274  Idx.getValueType()));
2275  ReplaceValueWith(SDValue(N, 0), Res);
2276  return SDValue();
2277  }
2278 
2279  }
2280  }
2281 
2282  // Bit-convert the input vector to the equivalent integer vector
2283  SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2284  EVT IVT = NewOp.getValueType().getVectorElementType();
2285 
2286  // Extract the element as an (bit-cast) integer value
2287  SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2288  NewOp, N->getOperand(1));
2289 
2290  // Convert the element to the desired FP type
2291  EVT VT = N->getValueType(0);
2292  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2293  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2294 }
2295 
2296 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2297 // needs promotion, so does the argument X. Note that Y, if needed, will be
2298 // handled during operand promotion.
2299 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2300  EVT VT = N->getValueType(0);
2301  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2302  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2303 
2304  SDValue Op1 = N->getOperand(1);
2305 
2306  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2307 }
2308 
2309 // Unary operation where the result and the operand have PromoteFloat type
2310 // action. Construct a new SDNode with the promoted float value of the old
2311 // operand.
2312 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2313  EVT VT = N->getValueType(0);
2314  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2315  SDValue Op = GetPromotedFloat(N->getOperand(0));
2316 
2317  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2318 }
2319 
2320 // Binary operations where the result and both operands have PromoteFloat type
2321 // action. Construct a new SDNode with the promoted float values of the old
2322 // operands.
2323 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2324  EVT VT = N->getValueType(0);
2325  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2326  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2327  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2328  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2329 }
2330 
2331 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2332  EVT VT = N->getValueType(0);
2333  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2334  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2335  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2336  SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2337 
2338  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2339 }
2340 
2341 // Promote the Float (first) operand and retain the Integer (second) operand
2342 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2343  EVT VT = N->getValueType(0);
2344  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2345  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2346  SDValue Op1 = N->getOperand(1);
2347 
2348  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2349 }
2350 
2351 // Explicit operation to reduce precision. Reduce the value to half precision
2352 // and promote it back to the legal type.
2353 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2354  SDLoc DL(N);
2355 
2356  SDValue Op = N->getOperand(0);
2357  EVT VT = N->getValueType(0);
2358  EVT OpVT = Op->getValueType(0);
2359  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2360  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2361 
2362  // Round promoted float to desired precision
2363  SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2364  // Promote it back to the legal output type
2365  return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2366 }
2367 
2368 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2369  LoadSDNode *L = cast<LoadSDNode>(N);
2370  EVT VT = N->getValueType(0);
2371 
2372  // Load the value as an integer value with the same number of bits.
2373  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2374  SDValue newL = DAG.getLoad(
2375  L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2376  L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2377  L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2378  // Legalize the chain result by replacing uses of the old value chain with the
2379  // new one
2380  ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2381 
2382  // Convert the integer value to the desired FP type
2383  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2384  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2385 }
2386 
2387 // Construct a new SELECT node with the promoted true- and false- values.
2388 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2389  SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2390  SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2391 
2392  return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2393  N->getOperand(0), TrueVal, FalseVal);
2394 }
2395 
2396 // Construct a new SELECT_CC node with the promoted true- and false- values.
2397 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2398 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2399  SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2400  SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2401 
2402  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2403  TrueVal.getNode()->getValueType(0), N->getOperand(0),
2404  N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2405 }
2406 
2407 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2408 // float type.
2409 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2410  SDLoc DL(N);
2411  EVT VT = N->getValueType(0);
2412  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2413  SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2414  // Round the value to the desired precision (that of the source type).
2415  return DAG.getNode(
2416  ISD::FP_EXTEND, DL, NVT,
2417  DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2418 }
2419 
2420 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2421  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2422  N->getValueType(0)));
2423 }
2424 
2425 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2426  EVT VT = N->getValueType(0);
2427 
2428  AtomicSDNode *AM = cast<AtomicSDNode>(N);
2429  SDLoc SL(N);
2430 
2431  SDValue CastVal = BitConvertToInteger(AM->getVal());
2432  EVT CastVT = CastVal.getValueType();
2433 
2434  SDValue NewAtomic
2435  = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2436  DAG.getVTList(CastVT, MVT::Other),
2437  { AM->getChain(), AM->getBasePtr(), CastVal },
2438  AM->getMemOperand());
2439 
2440  SDValue Result = NewAtomic;
2441 
2442  if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2443  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2444  Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2445  NewAtomic);
2446  }
2447 
2448  // Legalize the chain result by replacing uses of the old value chain with the
2449  // new one
2450  ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2451 
2452  return Result;
2453 
2454 }
2455 
2456 //===----------------------------------------------------------------------===//
2457 // Half Result Soft Promotion
2458 //===----------------------------------------------------------------------===//
2459 
2460 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2461  LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2462  N->dump(&DAG); dbgs() << "\n");
2463  SDValue R = SDValue();
2464 
2465  // See if the target wants to custom expand this node.
2466  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2467  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2468  return;
2469  }
2470 
2471  switch (N->getOpcode()) {
2472  default:
2473 #ifndef NDEBUG
2474  dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2475  N->dump(&DAG); dbgs() << "\n";
2476 #endif
2477  llvm_unreachable("Do not know how to soft promote this operator's result!");
2478 
2479  case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2480  case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2482  R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2483  case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2484  case ISD::STRICT_FP_ROUND:
2485  case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2486 
2487  // Unary FP Operations
2488  case ISD::FABS:
2489  case ISD::FCBRT:
2490  case ISD::FCEIL:
2491  case ISD::FCOS:
2492  case ISD::FEXP:
2493  case ISD::FEXP2:
2494  case ISD::FFLOOR:
2495  case ISD::FLOG:
2496  case ISD::FLOG2:
2497  case ISD::FLOG10:
2498  case ISD::FNEARBYINT:
2499  case ISD::FNEG:
2500  case ISD::FREEZE:
2501  case ISD::FRINT:
2502  case ISD::FROUND:
2503  case ISD::FROUNDEVEN:
2504  case ISD::FSIN:
2505  case ISD::FSQRT:
2506  case ISD::FTRUNC:
2507  case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2508 
2509  // Binary FP Operations
2510  case ISD::FADD:
2511  case ISD::FDIV:
2512  case ISD::FMAXIMUM:
2513  case ISD::FMINIMUM:
2514  case ISD::FMAXNUM:
2515  case ISD::FMINNUM:
2516  case ISD::FMUL:
2517  case ISD::FPOW:
2518  case ISD::FREM:
2519  case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2520 
2521  case ISD::FMA: // FMA is same as FMAD
2522  case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2523 
2524  case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2525 
2526  case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2527  case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2528  case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2529  case ISD::SINT_TO_FP:
2530  case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2531  case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2532  case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2533  }
2534 
2535  if (R.getNode())
2536  SetSoftPromotedHalf(SDValue(N, ResNo), R);
2537 }
2538 
2539 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2540  return BitConvertToInteger(N->getOperand(0));
2541 }
2542 
2543 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2544  ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2545 
2546  // Get the (bit-cast) APInt of the APFloat and build an integer constant
2547  return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2548  MVT::i16);
2549 }
2550 
2551 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2552  SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2553  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2554  NewOp.getValueType().getVectorElementType(), NewOp,
2555  N->getOperand(1));
2556 }
2557 
2558 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2559  SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2560  SDValue RHS = BitConvertToInteger(N->getOperand(1));
2561  SDLoc dl(N);
2562 
2563  EVT LVT = LHS.getValueType();
2564  EVT RVT = RHS.getValueType();
2565 
2566  unsigned LSize = LVT.getSizeInBits();
2567  unsigned RSize = RVT.getSizeInBits();
2568 
2569  // First get the sign bit of second operand.
2570  SDValue SignBit = DAG.getNode(
2571  ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2572  DAG.getConstant(RSize - 1, dl,
2573  TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2574  SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2575 
2576  // Shift right or sign-extend it if the two operands have different types.
2577  int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2578  if (SizeDiff > 0) {
2579  SignBit =
2580  DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2581  DAG.getConstant(SizeDiff, dl,
2582  TLI.getShiftAmountTy(SignBit.getValueType(),
2583  DAG.getDataLayout())));
2584  SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2585  } else if (SizeDiff < 0) {
2586  SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2587  SignBit =
2588  DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2589  DAG.getConstant(-SizeDiff, dl,
2590  TLI.getShiftAmountTy(SignBit.getValueType(),
2591  DAG.getDataLayout())));
2592  }
2593 
2594  // Clear the sign bit of the first operand.
2595  SDValue Mask = DAG.getNode(
2596  ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2597  DAG.getConstant(LSize - 1, dl,
2598  TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2599  Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2600  LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2601 
2602  // Or the value with the sign bit.
2603  return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2604 }
2605 
2606 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2607  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2608  SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2609  SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2610  SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2611  SDLoc dl(N);
2612 
2613  // Promote to the larger FP type.
2614  Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2615  Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2616  Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2);
2617 
2618  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2619 
2620  // Convert back to FP16 as an integer.
2621  return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2622 }
2623 
2624 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2625  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2626  SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2627  SDValue Op1 = N->getOperand(1);
2628  SDLoc dl(N);
2629 
2630  Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2631 
2632  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2633 
2634  // Convert back to FP16 as an integer.
2635  return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2636 }
2637 
2638 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2639  if (N->isStrictFPOpcode()) {
2640  SDValue Res =
2642  {N->getOperand(0), N->getOperand(1)});
2643  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2644  return Res;
2645  }
2646 
2647  return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0));
2648 }
2649 
2650 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2651  LoadSDNode *L = cast<LoadSDNode>(N);
2652 
2653  // Load the value as an integer value with the same number of bits.
2654  assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2655  SDValue NewL =
2657  SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2659  L->getMemOperand()->getFlags(), L->getAAInfo());
2660  // Legalize the chain result by replacing uses of the old value chain with the
2661  // new one
2662  ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2663  return NewL;
2664 }
2665 
2666 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2667  SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2668  SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2669  return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2670  Op2);
2671 }
2672 
2673 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2674  SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2675  SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2676  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2677  N->getOperand(0), N->getOperand(1), Op2, Op3,
2678  N->getOperand(4));
2679 }
2680 
2681 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2682  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2683  SDLoc dl(N);
2684 
2685  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2686 
2687  // Round the value to the softened type.
2688  return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2689 }
2690 
2691 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2692  return DAG.getUNDEF(MVT::i16);
2693 }
2694 
2695 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2696  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2697  SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2698  SDLoc dl(N);
2699 
2700  // Promote to the larger FP type.
2701  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2702 
2703  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2704 
2705  // Convert back to FP16 as an integer.
2706  return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2707 }
2708 
2709 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2710  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2711  SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2712  SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2713  SDLoc dl(N);
2714 
2715  // Promote to the larger FP type.
2716  Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2717  Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2718 
2719  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2720 
2721  // Convert back to FP16 as an integer.
2722  return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2723 }
2724 
2725 //===----------------------------------------------------------------------===//
2726 // Half Operand Soft Promotion
2727 //===----------------------------------------------------------------------===//
2728 
2729 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2730  LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2731  N->dump(&DAG); dbgs() << "\n");
2732  SDValue Res = SDValue();
2733 
2734  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2735  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2736  return false;
2737  }
2738 
2739  // Nodes that use a promotion-requiring floating point operand, but doesn't
2740  // produce a soft promotion-requiring floating point result, need to be
2741  // legalized to use the soft promoted float operand. Nodes that produce at
2742  // least one soft promotion-requiring floating point result have their
2743  // operands legalized as a part of PromoteFloatResult.
2744  switch (N->getOpcode()) {
2745  default:
2746  #ifndef NDEBUG
2747  dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2748  N->dump(&DAG); dbgs() << "\n";
2749  #endif
2750  llvm_unreachable("Do not know how to soft promote this operator's operand!");
2751 
2752  case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2753  case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2754  case ISD::FP_TO_SINT:
2755  case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2756  case ISD::STRICT_FP_EXTEND:
2757  case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2758  case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2759  case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2760  case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2761  }
2762 
2763  if (!Res.getNode())
2764  return false;
2765 
2766  assert(Res.getNode() != N && "Expected a new node!");
2767 
2768  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2769  "Invalid operand expansion");
2770 
2771  ReplaceValueWith(SDValue(N, 0), Res);
2772  return false;
2773 }
2774 
2775 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2776  SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2777 
2778  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2779 }
2780 
2781 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2782  unsigned OpNo) {
2783  assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2784  SDValue Op1 = N->getOperand(1);
2785  SDLoc dl(N);
2786 
2787  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2788 
2789  Op1 = GetSoftPromotedHalf(Op1);
2790  Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2791 
2792  return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2793  Op1);
2794 }
2795 
2796 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2797  bool IsStrict = N->isStrictFPOpcode();
2798  SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
2799 
2800  if (IsStrict) {
2801  SDValue Res =
2803  {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
2804  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2805  ReplaceValueWith(SDValue(N, 0), Res);
2806  return SDValue();
2807  }
2808 
2809  return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op);
2810 }
2811 
2812 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
2813  SDValue Op = N->getOperand(0);
2814  SDLoc dl(N);
2815 
2816  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2817 
2818  Op = GetSoftPromotedHalf(Op);
2819 
2820  SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2821 
2822  return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
2823 }
2824 
2825 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
2826  unsigned OpNo) {
2827  assert(OpNo == 0 && "Can only soften the comparison values");
2828  SDValue Op0 = N->getOperand(0);
2829  SDValue Op1 = N->getOperand(1);
2830  SDLoc dl(N);
2831 
2832  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2833 
2834  Op0 = GetSoftPromotedHalf(Op0);
2835  Op1 = GetSoftPromotedHalf(Op1);
2836 
2837  // Promote to the larger FP type.
2838  Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2839  Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2840 
2841  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
2842  N->getOperand(2), N->getOperand(3), N->getOperand(4));
2843 }
2844 
2845 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
2846  SDValue Op0 = N->getOperand(0);
2847  SDValue Op1 = N->getOperand(1);
2848  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2849  SDLoc dl(N);
2850 
2851  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2852 
2853  Op0 = GetSoftPromotedHalf(Op0);
2854  Op1 = GetSoftPromotedHalf(Op1);
2855 
2856  // Promote to the larger FP type.
2857  Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2858  Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2859 
2860  return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
2861 }
2862 
2863 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
2864  assert(OpNo == 1 && "Can only soften the stored value!");
2865  StoreSDNode *ST = cast<StoreSDNode>(N);
2866  SDValue Val = ST->getValue();
2867  SDLoc dl(N);
2868 
2869  assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
2870  SDValue Promoted = GetSoftPromotedHalf(Val);
2871  return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
2872  ST->getMemOperand());
2873 }
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:744
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:717
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:794
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOffset() const
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:363
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:206
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:566
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
DiagnosticInfoOptimizationBase::Argument NV
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:854
This class represents lattice values for constants.
Definition: AllocatorList.h:23
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0...
Definition: ISDOpcodes.h:807
const SDValue & getVal() const
This structure is used to pass arguments to makeLibCall function.
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
const SDValue & getValue() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:260
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
const SDValue & getBasePtr() const
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:352
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
const SDNodeFlags getFlags() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:217
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:612
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Shift and rotation operations.
Definition: ISDOpcodes.h:576
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:213
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
SimpleValueType SimpleTy
The memory access is dereferenceable (i.e., doesn&#39;t trap).
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:1022
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:424
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This is an SDNode representing atomic operations.
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only...
Definition: ISDOpcodes.h:427
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:657
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1513
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:400
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:926
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1204
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:421
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:195
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:703
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified, possibly variable, elements.
Definition: ISDOpcodes.h:456
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:242
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Simple binary floating point operators.
Definition: ISDOpcodes.h:342
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:280
const SDValue & getOperand(unsigned Num) const
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:254
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:923
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:729
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:315
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:559
void dump() const
Dump this node, for debugging.
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:448
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:476
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:272
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
MakeLibCallOptions & setSExt(bool Value=true)
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:754
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
EVT getMemoryVT() const
Return the type of the in-memory value.
Class for arbitrary precision integers.
Definition: APInt.h:69
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:597
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:647
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:650
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:441
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:693
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:178
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:551
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:817
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
#define N
Flags getFlags() const
Return the raw flags of the source value,.
The memory access always returns the same value (or traps).
uint32_t Size
Definition: Profile.cpp:46
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:416
SDValue getValue(unsigned R) const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:431
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC&#39;s if you just have an ISD::CondCode instead of an...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
const APFloat & getValueAPF() const
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations...
Definition: ISDOpcodes.h:435
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:620
APInt bitcastToAPInt() const
Definition: APFloat.h:1130
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:220
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
Conversion operators.
Definition: ISDOpcodes.h:644
bool isBigEndian() const
Definition: DataLayout.h:234
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:653
#define LLVM_DEBUG(X)
Definition: Debug.h:122
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:762
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:393
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
LLVMContext * getContext() const
Definition: SelectionDAG.h:431
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This class is used to represent ISD::LOAD nodes.