LLVM  6.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/Loads.h"
27 #include "llvm/CodeGen/Analysis.h"
28 #include "llvm/CodeGen/FastISel.h"
40 #include "llvm/CodeGen/StackMaps.h"
42 #include "llvm/IR/CallingConv.h"
43 #include "llvm/IR/ConstantRange.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DataLayout.h"
46 #include "llvm/IR/DebugInfo.h"
47 #include "llvm/IR/DerivedTypes.h"
48 #include "llvm/IR/Function.h"
50 #include "llvm/IR/GlobalVariable.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/Instructions.h"
53 #include "llvm/IR/IntrinsicInst.h"
54 #include "llvm/IR/Intrinsics.h"
55 #include "llvm/IR/LLVMContext.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Statepoint.h"
58 #include "llvm/MC/MCSymbol.h"
60 #include "llvm/Support/Debug.h"
70 #include <algorithm>
71 #include <utility>
72 using namespace llvm;
73 
74 #define DEBUG_TYPE "isel"
75 
76 /// LimitFloatPrecision - Generate low-precision inline sequences for
77 /// some float libcalls (6, 8 or 12 bits).
78 static unsigned LimitFloatPrecision;
79 
81 LimitFPPrecision("limit-float-precision",
82  cl::desc("Generate low-precision inline sequences "
83  "for some float libcalls"),
84  cl::location(LimitFloatPrecision),
85  cl::init(0));
86 // Limit the width of DAG chains. This is important in general to prevent
87 // DAG-based analysis from blowing up. For example, alias analysis and
88 // load clustering may not complete in reasonable time. It is difficult to
89 // recognize and avoid this situation within each individual analysis, and
90 // future analyses are likely to have the same behavior. Limiting DAG width is
91 // the safe approach and will be especially important with global DAGs.
92 //
93 // MaxParallelChains default is arbitrarily high to avoid affecting
94 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
95 // sequence over this should have been converted to llvm.memcpy by the
96 // frontend. It is easy to induce this behavior with .ll code such as:
97 // %buffer = alloca [4096 x i8]
98 // %data = load [4096 x i8]* %argPtr
99 // store [4096 x i8] %data, [4096 x i8]* %buffer
100 static const unsigned MaxParallelChains = 64;
101 
102 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
103  const SDValue *Parts, unsigned NumParts,
104  MVT PartVT, EVT ValueVT, const Value *V,
105  bool IsABIRegCopy);
106 
107 /// getCopyFromParts - Create a value that contains the specified legal parts
108 /// combined into the value they represent. If the parts combine to a type
109 /// larger than ValueVT then AssertOp can be used to specify whether the extra
110 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
111 /// (ISD::AssertSext).
113  const SDValue *Parts, unsigned NumParts,
114  MVT PartVT, EVT ValueVT, const Value *V,
115  Optional<ISD::NodeType> AssertOp = None,
116  bool IsABIRegCopy = false) {
117  if (ValueVT.isVector())
118  return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
119  PartVT, ValueVT, V, IsABIRegCopy);
120 
121  assert(NumParts > 0 && "No parts to assemble!");
122  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
123  SDValue Val = Parts[0];
124 
125  if (NumParts > 1) {
126  // Assemble the value from multiple parts.
127  if (ValueVT.isInteger()) {
128  unsigned PartBits = PartVT.getSizeInBits();
129  unsigned ValueBits = ValueVT.getSizeInBits();
130 
131  // Assemble the power of 2 part.
132  unsigned RoundParts = NumParts & (NumParts - 1) ?
133  1 << Log2_32(NumParts) : NumParts;
134  unsigned RoundBits = PartBits * RoundParts;
135  EVT RoundVT = RoundBits == ValueBits ?
136  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
137  SDValue Lo, Hi;
138 
139  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
140 
141  if (RoundParts > 2) {
142  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
143  PartVT, HalfVT, V);
144  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
145  RoundParts / 2, PartVT, HalfVT, V);
146  } else {
147  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
148  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
149  }
150 
151  if (DAG.getDataLayout().isBigEndian())
152  std::swap(Lo, Hi);
153 
154  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
155 
156  if (RoundParts < NumParts) {
157  // Assemble the trailing non-power-of-2 part.
158  unsigned OddParts = NumParts - RoundParts;
159  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
160  Hi = getCopyFromParts(DAG, DL,
161  Parts + RoundParts, OddParts, PartVT, OddVT, V);
162 
163  // Combine the round and odd parts.
164  Lo = Val;
165  if (DAG.getDataLayout().isBigEndian())
166  std::swap(Lo, Hi);
167  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
168  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
169  Hi =
170  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
171  DAG.getConstant(Lo.getValueSizeInBits(), DL,
172  TLI.getPointerTy(DAG.getDataLayout())));
173  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
174  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
175  }
176  } else if (PartVT.isFloatingPoint()) {
177  // FP split into multiple FP parts (for ppcf128)
178  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
179  "Unexpected split");
180  SDValue Lo, Hi;
181  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
182  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
183  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
184  std::swap(Lo, Hi);
185  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
186  } else {
187  // FP split into integer parts (soft fp)
188  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
189  !PartVT.isVector() && "Unexpected split");
190  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
191  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
192  }
193  }
194 
195  // There is now one part, held in Val. Correct it to match ValueVT.
196  // PartEVT is the type of the register class that holds the value.
197  // ValueVT is the type of the inline asm operation.
198  EVT PartEVT = Val.getValueType();
199 
200  if (PartEVT == ValueVT)
201  return Val;
202 
203  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
204  ValueVT.bitsLT(PartEVT)) {
205  // For an FP value in an integer part, we need to truncate to the right
206  // width first.
207  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
208  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
209  }
210 
211  // Handle types that have the same size.
212  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
213  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
214 
215  // Handle types with different sizes.
216  if (PartEVT.isInteger() && ValueVT.isInteger()) {
217  if (ValueVT.bitsLT(PartEVT)) {
218  // For a truncate, see if we have any information to
219  // indicate whether the truncated bits will always be
220  // zero or sign-extension.
221  if (AssertOp.hasValue())
222  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
223  DAG.getValueType(ValueVT));
224  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
225  }
226  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
227  }
228 
229  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
230  // FP_ROUND's are always exact here.
231  if (ValueVT.bitsLT(Val.getValueType()))
232  return DAG.getNode(
233  ISD::FP_ROUND, DL, ValueVT, Val,
234  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
235 
236  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
237  }
238 
239  llvm_unreachable("Unknown mismatch!");
240 }
241 
243  const Twine &ErrMsg) {
244  const Instruction *I = dyn_cast_or_null<Instruction>(V);
245  if (!V)
246  return Ctx.emitError(ErrMsg);
247 
248  const char *AsmError = ", possible invalid constraint for vector type";
249  if (const CallInst *CI = dyn_cast<CallInst>(I))
250  if (isa<InlineAsm>(CI->getCalledValue()))
251  return Ctx.emitError(I, ErrMsg + AsmError);
252 
253  return Ctx.emitError(I, ErrMsg);
254 }
255 
256 /// getCopyFromPartsVector - Create a value that contains the specified legal
257 /// parts combined into the value they represent. If the parts combine to a
258 /// type larger than ValueVT then AssertOp can be used to specify whether the
259 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
260 /// ValueVT (ISD::AssertSext).
262  const SDValue *Parts, unsigned NumParts,
263  MVT PartVT, EVT ValueVT, const Value *V,
264  bool IsABIRegCopy) {
265  assert(ValueVT.isVector() && "Not a vector value");
266  assert(NumParts > 0 && "No parts to assemble!");
267  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
268  SDValue Val = Parts[0];
269 
270  // Handle a multi-element vector.
271  if (NumParts > 1) {
272  EVT IntermediateVT;
273  MVT RegisterVT;
274  unsigned NumIntermediates;
275  unsigned NumRegs;
276 
277  if (IsABIRegCopy) {
279  *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
280  RegisterVT);
281  } else {
282  NumRegs =
283  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
284  NumIntermediates, RegisterVT);
285  }
286 
287  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
288  NumParts = NumRegs; // Silence a compiler warning.
289  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
290  assert(RegisterVT.getSizeInBits() ==
291  Parts[0].getSimpleValueType().getSizeInBits() &&
292  "Part type sizes don't match!");
293 
294  // Assemble the parts into intermediate operands.
295  SmallVector<SDValue, 8> Ops(NumIntermediates);
296  if (NumIntermediates == NumParts) {
297  // If the register was not expanded, truncate or copy the value,
298  // as appropriate.
299  for (unsigned i = 0; i != NumParts; ++i)
300  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
301  PartVT, IntermediateVT, V);
302  } else if (NumParts > 0) {
303  // If the intermediate type was expanded, build the intermediate
304  // operands from the parts.
305  assert(NumParts % NumIntermediates == 0 &&
306  "Must expand into a divisible number of parts!");
307  unsigned Factor = NumParts / NumIntermediates;
308  for (unsigned i = 0; i != NumIntermediates; ++i)
309  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
310  PartVT, IntermediateVT, V);
311  }
312 
313  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
314  // intermediate operands.
315  EVT BuiltVectorTy =
316  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
317  (IntermediateVT.isVector()
318  ? IntermediateVT.getVectorNumElements() * NumParts
319  : NumIntermediates));
320  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
322  DL, BuiltVectorTy, Ops);
323  }
324 
325  // There is now one part, held in Val. Correct it to match ValueVT.
326  EVT PartEVT = Val.getValueType();
327 
328  if (PartEVT == ValueVT)
329  return Val;
330 
331  if (PartEVT.isVector()) {
332  // If the element type of the source/dest vectors are the same, but the
333  // parts vector has more elements than the value vector, then we have a
334  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
335  // elements we want.
336  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
337  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
338  "Cannot narrow, it would be a lossy transformation");
339  return DAG.getNode(
340  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
341  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
342  }
343 
344  // Vector/Vector bitcast.
345  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
346  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
347 
348  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
349  "Cannot handle this kind of promotion");
350  // Promoted vector extract
351  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
352 
353  }
354 
355  // Trivial bitcast if the types are the same size and the destination
356  // vector type is legal.
357  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
358  TLI.isTypeLegal(ValueVT))
359  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
360 
361  if (ValueVT.getVectorNumElements() != 1) {
362  // Certain ABIs require that vectors are passed as integers. For vectors
363  // are the same size, this is an obvious bitcast.
364  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
365  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
366  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
367  // Bitcast Val back the original type and extract the corresponding
368  // vector we want.
369  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
370  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
371  ValueVT.getVectorElementType(), Elts);
372  Val = DAG.getBitcast(WiderVecType, Val);
373  return DAG.getNode(
374  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
375  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
376  }
377 
379  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
380  return DAG.getUNDEF(ValueVT);
381  }
382 
383  // Handle cases such as i8 -> <1 x i1>
384  EVT ValueSVT = ValueVT.getVectorElementType();
385  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
386  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
387  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
388 
389  return DAG.getBuildVector(ValueVT, DL, Val);
390 }
391 
392 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
393  SDValue Val, SDValue *Parts, unsigned NumParts,
394  MVT PartVT, const Value *V, bool IsABIRegCopy);
395 
396 /// getCopyToParts - Create a series of nodes that contain the specified value
397 /// split into legal parts. If the parts contain more bits than Val, then, for
398 /// integers, ExtendKind can be used to specify how to generate the extra bits.
399 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
400  SDValue *Parts, unsigned NumParts, MVT PartVT,
401  const Value *V,
402  ISD::NodeType ExtendKind = ISD::ANY_EXTEND,
403  bool IsABIRegCopy = false) {
404  EVT ValueVT = Val.getValueType();
405 
406  // Handle the vector case separately.
407  if (ValueVT.isVector())
408  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
409  IsABIRegCopy);
410 
411  unsigned PartBits = PartVT.getSizeInBits();
412  unsigned OrigNumParts = NumParts;
413  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
414  "Copying to an illegal type!");
415 
416  if (NumParts == 0)
417  return;
418 
419  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
420  EVT PartEVT = PartVT;
421  if (PartEVT == ValueVT) {
422  assert(NumParts == 1 && "No-op copy with multiple parts!");
423  Parts[0] = Val;
424  return;
425  }
426 
427  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
428  // If the parts cover more bits than the value has, promote the value.
429  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
430  assert(NumParts == 1 && "Do not know what to promote to!");
431  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
432  } else {
433  if (ValueVT.isFloatingPoint()) {
434  // FP values need to be bitcast, then extended if they are being put
435  // into a larger container.
436  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
437  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
438  }
439  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
440  ValueVT.isInteger() &&
441  "Unknown mismatch!");
442  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
443  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
444  if (PartVT == MVT::x86mmx)
445  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
446  }
447  } else if (PartBits == ValueVT.getSizeInBits()) {
448  // Different types of the same size.
449  assert(NumParts == 1 && PartEVT != ValueVT);
450  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
451  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
452  // If the parts cover less bits than value has, truncate the value.
453  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
454  ValueVT.isInteger() &&
455  "Unknown mismatch!");
456  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
457  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
458  if (PartVT == MVT::x86mmx)
459  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
460  }
461 
462  // The value may have changed - recompute ValueVT.
463  ValueVT = Val.getValueType();
464  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
465  "Failed to tile the value with PartVT!");
466 
467  if (NumParts == 1) {
468  if (PartEVT != ValueVT) {
470  "scalar-to-vector conversion failed");
471  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
472  }
473 
474  Parts[0] = Val;
475  return;
476  }
477 
478  // Expand the value into multiple parts.
479  if (NumParts & (NumParts - 1)) {
480  // The number of parts is not a power of 2. Split off and copy the tail.
481  assert(PartVT.isInteger() && ValueVT.isInteger() &&
482  "Do not know what to expand to!");
483  unsigned RoundParts = 1 << Log2_32(NumParts);
484  unsigned RoundBits = RoundParts * PartBits;
485  unsigned OddParts = NumParts - RoundParts;
486  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
487  DAG.getIntPtrConstant(RoundBits, DL));
488  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
489 
490  if (DAG.getDataLayout().isBigEndian())
491  // The odd parts were reversed by getCopyToParts - unreverse them.
492  std::reverse(Parts + RoundParts, Parts + NumParts);
493 
494  NumParts = RoundParts;
495  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
496  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
497  }
498 
499  // The number of parts is a power of 2. Repeatedly bisect the value using
500  // EXTRACT_ELEMENT.
501  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
503  ValueVT.getSizeInBits()),
504  Val);
505 
506  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
507  for (unsigned i = 0; i < NumParts; i += StepSize) {
508  unsigned ThisBits = StepSize * PartBits / 2;
509  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
510  SDValue &Part0 = Parts[i];
511  SDValue &Part1 = Parts[i+StepSize/2];
512 
513  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
514  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
515  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
516  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
517 
518  if (ThisBits == PartBits && ThisVT != PartVT) {
519  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
520  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
521  }
522  }
523  }
524 
525  if (DAG.getDataLayout().isBigEndian())
526  std::reverse(Parts, Parts + OrigNumParts);
527 }
528 
529 
530 /// getCopyToPartsVector - Create a series of nodes that contain the specified
531 /// value split into legal parts.
532 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
533  SDValue Val, SDValue *Parts, unsigned NumParts,
534  MVT PartVT, const Value *V,
535  bool IsABIRegCopy) {
536 
537  EVT ValueVT = Val.getValueType();
538  assert(ValueVT.isVector() && "Not a vector");
539  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
540 
541  if (NumParts == 1) {
542  EVT PartEVT = PartVT;
543  if (PartEVT == ValueVT) {
544  // Nothing to do.
545  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
546  // Bitconvert vector->vector case.
547  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
548  } else if (PartVT.isVector() &&
549  PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
550  PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
551  EVT ElementVT = PartVT.getVectorElementType();
552  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
553  // undef elements.
555  for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
556  Ops.push_back(DAG.getNode(
557  ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
558  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
559 
560  for (unsigned i = ValueVT.getVectorNumElements(),
561  e = PartVT.getVectorNumElements(); i != e; ++i)
562  Ops.push_back(DAG.getUNDEF(ElementVT));
563 
564  Val = DAG.getBuildVector(PartVT, DL, Ops);
565 
566  // FIXME: Use CONCAT for 2x -> 4x.
567 
568  //SDValue UndefElts = DAG.getUNDEF(VectorTy);
569  //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
570  } else if (PartVT.isVector() &&
571  PartEVT.getVectorElementType().bitsGE(
572  ValueVT.getVectorElementType()) &&
573  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
574 
575  // Promoted vector extract
576  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
577  } else {
578  if (ValueVT.getVectorNumElements() == 1) {
579  Val = DAG.getNode(
580  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
581  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
582 
583  } else {
584  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
585  "lossy conversion of vector to scalar type");
586  EVT IntermediateType =
587  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
588  Val = DAG.getBitcast(IntermediateType, Val);
589  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
590  }
591  }
592 
593  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
594  Parts[0] = Val;
595  return;
596  }
597 
598  // Handle a multi-element vector.
599  EVT IntermediateVT;
600  MVT RegisterVT;
601  unsigned NumIntermediates;
602  unsigned NumRegs;
603  if (IsABIRegCopy) {
604  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
605  *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
606  RegisterVT);
607  } else {
608  NumRegs =
609  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
610  NumIntermediates, RegisterVT);
611  }
612  unsigned NumElements = ValueVT.getVectorNumElements();
613 
614  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
615  NumParts = NumRegs; // Silence a compiler warning.
616  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
617 
618  // Convert the vector to the appropiate type if necessary.
619  unsigned DestVectorNoElts =
620  NumIntermediates *
621  (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1);
622  EVT BuiltVectorTy = EVT::getVectorVT(
623  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
624  if (Val.getValueType() != BuiltVectorTy)
625  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
626 
627  // Split the vector into intermediate operands.
628  SmallVector<SDValue, 8> Ops(NumIntermediates);
629  for (unsigned i = 0; i != NumIntermediates; ++i) {
630  if (IntermediateVT.isVector())
631  Ops[i] =
632  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
633  DAG.getConstant(i * (NumElements / NumIntermediates), DL,
634  TLI.getVectorIdxTy(DAG.getDataLayout())));
635  else
636  Ops[i] = DAG.getNode(
637  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
638  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
639  }
640 
641  // Split the intermediate operands into legal parts.
642  if (NumParts == NumIntermediates) {
643  // If the register was not expanded, promote or copy the value,
644  // as appropriate.
645  for (unsigned i = 0; i != NumParts; ++i)
646  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
647  } else if (NumParts > 0) {
648  // If the intermediate type was expanded, split each the value into
649  // legal parts.
650  assert(NumIntermediates != 0 && "division by zero");
651  assert(NumParts % NumIntermediates == 0 &&
652  "Must expand into a divisible number of parts!");
653  unsigned Factor = NumParts / NumIntermediates;
654  for (unsigned i = 0; i != NumIntermediates; ++i)
655  getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
656  }
657 }
658 
660 
662  EVT valuevt, bool IsABIMangledValue)
663  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
664  RegCount(1, regs.size()), IsABIMangled(IsABIMangledValue) {}
665 
667  const DataLayout &DL, unsigned Reg, Type *Ty,
668  bool IsABIMangledValue) {
669  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
670 
671  IsABIMangled = IsABIMangledValue;
672 
673  for (EVT ValueVT : ValueVTs) {
674  unsigned NumRegs = IsABIMangledValue
675  ? TLI.getNumRegistersForCallingConv(Context, ValueVT)
676  : TLI.getNumRegisters(Context, ValueVT);
677  MVT RegisterVT = IsABIMangledValue
678  ? TLI.getRegisterTypeForCallingConv(Context, ValueVT)
679  : TLI.getRegisterType(Context, ValueVT);
680  for (unsigned i = 0; i != NumRegs; ++i)
681  Regs.push_back(Reg + i);
682  RegVTs.push_back(RegisterVT);
683  RegCount.push_back(NumRegs);
684  Reg += NumRegs;
685  }
686 }
687 
689  FunctionLoweringInfo &FuncInfo,
690  const SDLoc &dl, SDValue &Chain,
691  SDValue *Flag, const Value *V) const {
692  // A Value with type {} or [0 x %t] needs no registers.
693  if (ValueVTs.empty())
694  return SDValue();
695 
696  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
697 
698  // Assemble the legal parts into the final values.
699  SmallVector<SDValue, 4> Values(ValueVTs.size());
701  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
702  // Copy the legal parts from the registers.
703  EVT ValueVT = ValueVTs[Value];
704  unsigned NumRegs = RegCount[Value];
705  MVT RegisterVT = IsABIMangled
707  : RegVTs[Value];
708 
709  Parts.resize(NumRegs);
710  for (unsigned i = 0; i != NumRegs; ++i) {
711  SDValue P;
712  if (!Flag) {
713  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
714  } else {
715  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
716  *Flag = P.getValue(2);
717  }
718 
719  Chain = P.getValue(1);
720  Parts[i] = P;
721 
722  // If the source register was virtual and if we know something about it,
723  // add an assert node.
725  !RegisterVT.isInteger() || RegisterVT.isVector())
726  continue;
727 
729  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
730  if (!LOI)
731  continue;
732 
733  unsigned RegSize = RegisterVT.getSizeInBits();
734  unsigned NumSignBits = LOI->NumSignBits;
735  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
736 
737  if (NumZeroBits == RegSize) {
738  // The current value is a zero.
739  // Explicitly express that as it would be easier for
740  // optimizations to kick in.
741  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
742  continue;
743  }
744 
745  // FIXME: We capture more information than the dag can represent. For
746  // now, just use the tightest assertzext/assertsext possible.
747  bool isSExt = true;
748  EVT FromVT(MVT::Other);
749  if (NumSignBits == RegSize) {
750  isSExt = true; // ASSERT SEXT 1
751  FromVT = MVT::i1;
752  } else if (NumZeroBits >= RegSize - 1) {
753  isSExt = false; // ASSERT ZEXT 1
754  FromVT = MVT::i1;
755  } else if (NumSignBits > RegSize - 8) {
756  isSExt = true; // ASSERT SEXT 8
757  FromVT = MVT::i8;
758  } else if (NumZeroBits >= RegSize - 8) {
759  isSExt = false; // ASSERT ZEXT 8
760  FromVT = MVT::i8;
761  } else if (NumSignBits > RegSize - 16) {
762  isSExt = true; // ASSERT SEXT 16
763  FromVT = MVT::i16;
764  } else if (NumZeroBits >= RegSize - 16) {
765  isSExt = false; // ASSERT ZEXT 16
766  FromVT = MVT::i16;
767  } else if (NumSignBits > RegSize - 32) {
768  isSExt = true; // ASSERT SEXT 32
769  FromVT = MVT::i32;
770  } else if (NumZeroBits >= RegSize - 32) {
771  isSExt = false; // ASSERT ZEXT 32
772  FromVT = MVT::i32;
773  } else {
774  continue;
775  }
776  // Add an assertion node.
777  assert(FromVT != MVT::Other);
778  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
779  RegisterVT, P, DAG.getValueType(FromVT));
780  }
781 
782  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
783  NumRegs, RegisterVT, ValueVT, V);
784  Part += NumRegs;
785  Parts.clear();
786  }
787 
788  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
789 }
790 
792  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
793  const Value *V,
794  ISD::NodeType PreferredExtendType) const {
795  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
796  ISD::NodeType ExtendKind = PreferredExtendType;
797 
798  // Get the list of the values's legal parts.
799  unsigned NumRegs = Regs.size();
800  SmallVector<SDValue, 8> Parts(NumRegs);
801  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
802  unsigned NumParts = RegCount[Value];
803 
804  MVT RegisterVT = IsABIMangled
806  : RegVTs[Value];
807 
808  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
809  ExtendKind = ISD::ZERO_EXTEND;
810 
811  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
812  &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
813  Part += NumParts;
814  }
815 
816  // Copy the parts into the registers.
817  SmallVector<SDValue, 8> Chains(NumRegs);
818  for (unsigned i = 0; i != NumRegs; ++i) {
819  SDValue Part;
820  if (!Flag) {
821  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
822  } else {
823  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
824  *Flag = Part.getValue(1);
825  }
826 
827  Chains[i] = Part.getValue(0);
828  }
829 
830  if (NumRegs == 1 || Flag)
831  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
832  // flagged to it. That is the CopyToReg nodes and the user are considered
833  // a single scheduling unit. If we create a TokenFactor and return it as
834  // chain, then the TokenFactor is both a predecessor (operand) of the
835  // user as well as a successor (the TF operands are flagged to the user).
836  // c1, f1 = CopyToReg
837  // c2, f2 = CopyToReg
838  // c3 = TokenFactor c1, c2
839  // ...
840  // = op c3, ..., f2
841  Chain = Chains[NumRegs-1];
842  else
843  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
844 }
845 
846 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
847  unsigned MatchingIdx, const SDLoc &dl,
848  SelectionDAG &DAG,
849  std::vector<SDValue> &Ops) const {
850  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
851 
852  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
853  if (HasMatching)
854  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
855  else if (!Regs.empty() &&
857  // Put the register class of the virtual registers in the flag word. That
858  // way, later passes can recompute register class constraints for inline
859  // assembly as well as normal instructions.
860  // Don't do this for tied operands that can use the regclass information
861  // from the def.
863  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
864  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
865  }
866 
867  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
868  Ops.push_back(Res);
869 
870  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
871  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
872  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
873  MVT RegisterVT = RegVTs[Value];
874  for (unsigned i = 0; i != NumRegs; ++i) {
875  assert(Reg < Regs.size() && "Mismatch in # registers expected");
876  unsigned TheReg = Regs[Reg++];
877  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
878 
879  if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
880  // If we clobbered the stack pointer, MFI should know about it.
882  }
883  }
884  }
885 }
886 
888  const TargetLibraryInfo *li) {
889  AA = aa;
890  GFI = gfi;
891  LibInfo = li;
892  DL = &DAG.getDataLayout();
893  Context = DAG.getContext();
894  LPadToCallSiteMap.clear();
895 }
896 
898  NodeMap.clear();
899  UnusedArgNodeMap.clear();
900  PendingLoads.clear();
901  PendingExports.clear();
902  CurInst = nullptr;
903  HasTailCall = false;
904  SDNodeOrder = LowestSDNodeOrder;
905  StatepointLowering.clear();
906 }
907 
909  DanglingDebugInfoMap.clear();
910 }
911 
913  if (PendingLoads.empty())
914  return DAG.getRoot();
915 
916  if (PendingLoads.size() == 1) {
917  SDValue Root = PendingLoads[0];
918  DAG.setRoot(Root);
919  PendingLoads.clear();
920  return Root;
921  }
922 
923  // Otherwise, we have to make a token factor node.
924  SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
925  PendingLoads);
926  PendingLoads.clear();
927  DAG.setRoot(Root);
928  return Root;
929 }
930 
932  SDValue Root = DAG.getRoot();
933 
934  if (PendingExports.empty())
935  return Root;
936 
937  // Turn all of the CopyToReg chains into one factored node.
938  if (Root.getOpcode() != ISD::EntryToken) {
939  unsigned i = 0, e = PendingExports.size();
940  for (; i != e; ++i) {
941  assert(PendingExports[i].getNode()->getNumOperands() > 1);
942  if (PendingExports[i].getNode()->getOperand(0) == Root)
943  break; // Don't add the root if we already indirectly depend on it.
944  }
945 
946  if (i == e)
947  PendingExports.push_back(Root);
948  }
949 
950  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
951  PendingExports);
952  PendingExports.clear();
953  DAG.setRoot(Root);
954  return Root;
955 }
956 
958  // Set up outgoing PHI node register values before emitting the terminator.
959  if (isa<TerminatorInst>(&I)) {
960  HandlePHINodesInSuccessorBlocks(I.getParent());
961  }
962 
963  // Increase the SDNodeOrder if dealing with a non-debug instruction.
964  if (!isa<DbgInfoIntrinsic>(I))
965  ++SDNodeOrder;
966 
967  CurInst = &I;
968 
969  visit(I.getOpcode(), I);
970 
971  if (!isa<TerminatorInst>(&I) && !HasTailCall &&
972  !isStatepoint(&I)) // statepoints handle their exports internally
973  CopyToExportRegsIfNeeded(&I);
974 
975  CurInst = nullptr;
976 }
977 
978 void SelectionDAGBuilder::visitPHI(const PHINode &) {
979  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
980 }
981 
982 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
983  // Note: this doesn't use InstVisitor, because it has to work with
984  // ConstantExpr's in addition to instructions.
985  switch (Opcode) {
986  default: llvm_unreachable("Unknown instruction type encountered!");
987  // Build the switch statement using the Instruction.def file.
988 #define HANDLE_INST(NUM, OPCODE, CLASS) \
989  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
990 #include "llvm/IR/Instruction.def"
991  }
992 }
993 
994 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
995 // generate the debug data structures now that we've seen its definition.
997  SDValue Val) {
998  DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
999  if (DDI.getDI()) {
1000  const DbgValueInst *DI = DDI.getDI();
1001  DebugLoc dl = DDI.getdl();
1002  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1003  DILocalVariable *Variable = DI->getVariable();
1004  DIExpression *Expr = DI->getExpression();
1005  assert(Variable->isValidLocationForIntrinsic(dl) &&
1006  "Expected inlined-at fields to agree");
1007  uint64_t Offset = DI->getOffset();
1008  SDDbgValue *SDV;
1009  if (Val.getNode()) {
1010  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
1011  Val)) {
1012  SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
1013  DAG.AddDbgValue(SDV, Val.getNode(), false);
1014  }
1015  } else
1016  DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1017  DanglingDebugInfoMap[V] = DanglingDebugInfo();
1018  }
1019 }
1020 
1021 /// getCopyFromRegs - If there was virtual register allocated for the value V
1022 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1024  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1025  SDValue Result;
1026 
1027  if (It != FuncInfo.ValueMap.end()) {
1028  unsigned InReg = It->second;
1029  bool IsABIRegCopy =
1030  V && ((isa<CallInst>(V) &&
1031  !(static_cast<const CallInst *>(V))->isInlineAsm()) ||
1032  isa<ReturnInst>(V));
1033 
1034  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1035  DAG.getDataLayout(), InReg, Ty, IsABIRegCopy);
1036  SDValue Chain = DAG.getEntryNode();
1037  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1038  V);
1039  resolveDanglingDebugInfo(V, Result);
1040  }
1041 
1042  return Result;
1043 }
1044 
1045 /// getValue - Return an SDValue for the given Value.
1047  // If we already have an SDValue for this value, use it. It's important
1048  // to do this first, so that we don't create a CopyFromReg if we already
1049  // have a regular SDValue.
1050  SDValue &N = NodeMap[V];
1051  if (N.getNode()) return N;
1052 
1053  // If there's a virtual register allocated and initialized for this
1054  // value, use it.
1055  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1056  return copyFromReg;
1057 
1058  // Otherwise create a new SDValue and remember it.
1059  SDValue Val = getValueImpl(V);
1060  NodeMap[V] = Val;
1061  resolveDanglingDebugInfo(V, Val);
1062  return Val;
1063 }
1064 
1065 // Return true if SDValue exists for the given Value
1067  return (NodeMap.find(V) != NodeMap.end()) ||
1068  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1069 }
1070 
1071 /// getNonRegisterValue - Return an SDValue for the given Value, but
1072 /// don't look in FuncInfo.ValueMap for a virtual register.
1074  // If we already have an SDValue for this value, use it.
1075  SDValue &N = NodeMap[V];
1076  if (N.getNode()) {
1077  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1078  // Remove the debug location from the node as the node is about to be used
1079  // in a location which may differ from the original debug location. This
1080  // is relevant to Constant and ConstantFP nodes because they can appear
1081  // as constant expressions inside PHI nodes.
1082  N->setDebugLoc(DebugLoc());
1083  }
1084  return N;
1085  }
1086 
1087  // Otherwise create a new SDValue and remember it.
1088  SDValue Val = getValueImpl(V);
1089  NodeMap[V] = Val;
1090  resolveDanglingDebugInfo(V, Val);
1091  return Val;
1092 }
1093 
1094 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1095 /// Create an SDValue for the given value.
1097  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1098 
1099  if (const Constant *C = dyn_cast<Constant>(V)) {
1100  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1101 
1102  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1103  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1104 
1105  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1106  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1107 
1108  if (isa<ConstantPointerNull>(C)) {
1109  unsigned AS = V->getType()->getPointerAddressSpace();
1110  return DAG.getConstant(0, getCurSDLoc(),
1111  TLI.getPointerTy(DAG.getDataLayout(), AS));
1112  }
1113 
1114  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1115  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1116 
1117  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1118  return DAG.getUNDEF(VT);
1119 
1120  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1121  visit(CE->getOpcode(), *CE);
1122  SDValue N1 = NodeMap[V];
1123  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1124  return N1;
1125  }
1126 
1127  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1129  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1130  OI != OE; ++OI) {
1131  SDNode *Val = getValue(*OI).getNode();
1132  // If the operand is an empty aggregate, there are no values.
1133  if (!Val) continue;
1134  // Add each leaf value from the operand to the Constants list
1135  // to form a flattened list of all the values.
1136  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1137  Constants.push_back(SDValue(Val, i));
1138  }
1139 
1140  return DAG.getMergeValues(Constants, getCurSDLoc());
1141  }
1142 
1143  if (const ConstantDataSequential *CDS =
1144  dyn_cast<ConstantDataSequential>(C)) {
1146  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1147  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1148  // Add each leaf value from the operand to the Constants list
1149  // to form a flattened list of all the values.
1150  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1151  Ops.push_back(SDValue(Val, i));
1152  }
1153 
1154  if (isa<ArrayType>(CDS->getType()))
1155  return DAG.getMergeValues(Ops, getCurSDLoc());
1156  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1157  }
1158 
1159  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1160  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1161  "Unknown struct or array constant!");
1162 
1164  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1165  unsigned NumElts = ValueVTs.size();
1166  if (NumElts == 0)
1167  return SDValue(); // empty struct
1169  for (unsigned i = 0; i != NumElts; ++i) {
1170  EVT EltVT = ValueVTs[i];
1171  if (isa<UndefValue>(C))
1172  Constants[i] = DAG.getUNDEF(EltVT);
1173  else if (EltVT.isFloatingPoint())
1174  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1175  else
1176  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1177  }
1178 
1179  return DAG.getMergeValues(Constants, getCurSDLoc());
1180  }
1181 
1182  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1183  return DAG.getBlockAddress(BA, VT);
1184 
1185  VectorType *VecTy = cast<VectorType>(V->getType());
1186  unsigned NumElements = VecTy->getNumElements();
1187 
1188  // Now that we know the number and type of the elements, get that number of
1189  // elements into the Ops array based on what kind of constant it is.
1191  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1192  for (unsigned i = 0; i != NumElements; ++i)
1193  Ops.push_back(getValue(CV->getOperand(i)));
1194  } else {
1195  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1196  EVT EltVT =
1197  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1198 
1199  SDValue Op;
1200  if (EltVT.isFloatingPoint())
1201  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1202  else
1203  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1204  Ops.assign(NumElements, Op);
1205  }
1206 
1207  // Create a BUILD_VECTOR node.
1208  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1209  }
1210 
1211  // If this is a static alloca, generate it as the frameindex instead of
1212  // computation.
1213  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1215  FuncInfo.StaticAllocaMap.find(AI);
1216  if (SI != FuncInfo.StaticAllocaMap.end())
1217  return DAG.getFrameIndex(SI->second,
1218  TLI.getFrameIndexTy(DAG.getDataLayout()));
1219  }
1220 
1221  // If this is an instruction which fast-isel has deferred, select it now.
1222  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1223  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1224  bool IsABIRegCopy =
1225  V && ((isa<CallInst>(V) &&
1226  !(static_cast<const CallInst *>(V))->isInlineAsm()) ||
1227  isa<ReturnInst>(V));
1228 
1229  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1230  Inst->getType(), IsABIRegCopy);
1231  SDValue Chain = DAG.getEntryNode();
1232  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1233  }
1234 
1235  llvm_unreachable("Can't get register for value!");
1236 }
1237 
1238 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1239  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1240  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1241  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1242  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1243  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1244  if (IsMSVCCXX || IsCoreCLR)
1245  CatchPadMBB->setIsEHFuncletEntry();
1246 
1247  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
1248 }
1249 
1250 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1251  // Update machine-CFG edge.
1252  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1253  FuncInfo.MBB->addSuccessor(TargetMBB);
1254 
1255  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1256  bool IsSEH = isAsynchronousEHPersonality(Pers);
1257  if (IsSEH) {
1258  // If this is not a fall-through branch or optimizations are switched off,
1259  // emit the branch.
1260  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1261  TM.getOptLevel() == CodeGenOpt::None)
1262  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1263  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1264  return;
1265  }
1266 
1267  // Figure out the funclet membership for the catchret's successor.
1268  // This will be used by the FuncletLayout pass to determine how to order the
1269  // BB's.
1270  // A 'catchret' returns to the outer scope's color.
1271  Value *ParentPad = I.getCatchSwitchParentPad();
1272  const BasicBlock *SuccessorColor;
1273  if (isa<ConstantTokenNone>(ParentPad))
1274  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1275  else
1276  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1277  assert(SuccessorColor && "No parent funclet for catchret!");
1278  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1279  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1280 
1281  // Create the terminator node.
1282  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1283  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1284  DAG.getBasicBlock(SuccessorColorMBB));
1285  DAG.setRoot(Ret);
1286 }
1287 
1288 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1289  // Don't emit any special code for the cleanuppad instruction. It just marks
1290  // the start of a funclet.
1291  FuncInfo.MBB->setIsEHFuncletEntry();
1292  FuncInfo.MBB->setIsCleanupFuncletEntry();
1293 }
1294 
1295 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1296 /// many places it could ultimately go. In the IR, we have a single unwind
1297 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1298 /// This function skips over imaginary basic blocks that hold catchswitch
1299 /// instructions, and finds all the "real" machine
1300 /// basic block destinations. As those destinations may not be successors of
1301 /// EHPadBB, here we also calculate the edge probability to those destinations.
1302 /// The passed-in Prob is the edge probability to EHPadBB.
1304  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1305  BranchProbability Prob,
1306  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1307  &UnwindDests) {
1308  EHPersonality Personality =
1310  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1311  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1312 
1313  while (EHPadBB) {
1314  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1315  BasicBlock *NewEHPadBB = nullptr;
1316  if (isa<LandingPadInst>(Pad)) {
1317  // Stop on landingpads. They are not funclets.
1318  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1319  break;
1320  } else if (isa<CleanupPadInst>(Pad)) {
1321  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1322  // personalities.
1323  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1324  UnwindDests.back().first->setIsEHFuncletEntry();
1325  break;
1326  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1327  // Add the catchpad handlers to the possible destinations.
1328  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1329  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1330  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1331  if (IsMSVCCXX || IsCoreCLR)
1332  UnwindDests.back().first->setIsEHFuncletEntry();
1333  }
1334  NewEHPadBB = CatchSwitch->getUnwindDest();
1335  } else {
1336  continue;
1337  }
1338 
1339  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1340  if (BPI && NewEHPadBB)
1341  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1342  EHPadBB = NewEHPadBB;
1343  }
1344 }
1345 
1346 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1347  // Update successor info.
1349  auto UnwindDest = I.getUnwindDest();
1350  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1351  BranchProbability UnwindDestProb =
1352  (BPI && UnwindDest)
1353  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1355  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1356  for (auto &UnwindDest : UnwindDests) {
1357  UnwindDest.first->setIsEHPad();
1358  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1359  }
1360  FuncInfo.MBB->normalizeSuccProbs();
1361 
1362  // Create the terminator node.
1363  SDValue Ret =
1364  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1365  DAG.setRoot(Ret);
1366 }
1367 
1368 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1369  report_fatal_error("visitCatchSwitch not yet implemented!");
1370 }
1371 
1372 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1373  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1374  auto &DL = DAG.getDataLayout();
1375  SDValue Chain = getControlRoot();
1377  SmallVector<SDValue, 8> OutVals;
1378 
1379  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1380  // lower
1381  //
1382  // %val = call <ty> @llvm.experimental.deoptimize()
1383  // ret <ty> %val
1384  //
1385  // differently.
1387  LowerDeoptimizingReturn();
1388  return;
1389  }
1390 
1391  if (!FuncInfo.CanLowerReturn) {
1392  unsigned DemoteReg = FuncInfo.DemoteRegister;
1393  const Function *F = I.getParent()->getParent();
1394 
1395  // Emit a store of the return value through the virtual register.
1396  // Leave Outs empty so that LowerReturn won't try to load return
1397  // registers the usual way.
1398  SmallVector<EVT, 1> PtrValueVTs;
1400  PtrValueVTs);
1401 
1402  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1403  DemoteReg, PtrValueVTs[0]);
1404  SDValue RetOp = getValue(I.getOperand(0));
1405 
1408  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1409  unsigned NumValues = ValueVTs.size();
1410 
1411  // An aggregate return value cannot wrap around the address space, so
1412  // offsets to its parts don't wrap either.
1414  Flags.setNoUnsignedWrap(true);
1415 
1416  SmallVector<SDValue, 4> Chains(NumValues);
1417  for (unsigned i = 0; i != NumValues; ++i) {
1418  SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
1419  RetPtr.getValueType(), RetPtr,
1420  DAG.getIntPtrConstant(Offsets[i],
1421  getCurSDLoc()),
1422  Flags);
1423  Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1424  SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1425  // FIXME: better loc info would be nice.
1426  Add, MachinePointerInfo());
1427  }
1428 
1429  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1430  MVT::Other, Chains);
1431  } else if (I.getNumOperands() != 0) {
1433  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1434  unsigned NumValues = ValueVTs.size();
1435  if (NumValues) {
1436  SDValue RetOp = getValue(I.getOperand(0));
1437 
1438  const Function *F = I.getParent()->getParent();
1439 
1440  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1441  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1442  Attribute::SExt))
1443  ExtendKind = ISD::SIGN_EXTEND;
1444  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1445  Attribute::ZExt))
1446  ExtendKind = ISD::ZERO_EXTEND;
1447 
1448  LLVMContext &Context = F->getContext();
1449  bool RetInReg = F->getAttributes().hasAttribute(
1450  AttributeList::ReturnIndex, Attribute::InReg);
1451 
1452  for (unsigned j = 0; j != NumValues; ++j) {
1453  EVT VT = ValueVTs[j];
1454 
1455  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1456  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1457 
1458  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, VT);
1459  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, VT);
1460  SmallVector<SDValue, 4> Parts(NumParts);
1461  getCopyToParts(DAG, getCurSDLoc(),
1462  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1463  &Parts[0], NumParts, PartVT, &I, ExtendKind, true);
1464 
1465  // 'inreg' on function refers to return value
1467  if (RetInReg)
1468  Flags.setInReg();
1469 
1470  // Propagate extension type if any
1471  if (ExtendKind == ISD::SIGN_EXTEND)
1472  Flags.setSExt();
1473  else if (ExtendKind == ISD::ZERO_EXTEND)
1474  Flags.setZExt();
1475 
1476  for (unsigned i = 0; i < NumParts; ++i) {
1477  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1478  VT, /*isfixed=*/true, 0, 0));
1479  OutVals.push_back(Parts[i]);
1480  }
1481  }
1482  }
1483  }
1484 
1485  // Push in swifterror virtual register as the last element of Outs. This makes
1486  // sure swifterror virtual register will be returned in the swifterror
1487  // physical register.
1488  const Function *F = I.getParent()->getParent();
1489  if (TLI.supportSwiftError() &&
1490  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1491  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1493  Flags.setSwiftError();
1494  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1495  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1496  true /*isfixed*/, 1 /*origidx*/,
1497  0 /*partOffs*/));
1498  // Create SDNode for the swifterror virtual register.
1499  OutVals.push_back(
1500  DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1501  &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1502  EVT(TLI.getPointerTy(DL))));
1503  }
1504 
1505  bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1506  CallingConv::ID CallConv =
1508  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1509  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1510 
1511  // Verify that the target's LowerReturn behaved as expected.
1512  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1513  "LowerReturn didn't return a valid chain!");
1514 
1515  // Update the DAG with the new chain value resulting from return lowering.
1516  DAG.setRoot(Chain);
1517 }
1518 
1519 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1520 /// created for it, emit nodes to copy the value into the virtual
1521 /// registers.
1523  // Skip empty types
1524  if (V->getType()->isEmptyTy())
1525  return;
1526 
1527  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1528  if (VMI != FuncInfo.ValueMap.end()) {
1529  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1530  CopyValueToVirtualRegister(V, VMI->second);
1531  }
1532 }
1533 
1534 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1535 /// the current basic block, add it to ValueMap now so that we'll get a
1536 /// CopyTo/FromReg.
1538  // No need to export constants.
1539  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1540 
1541  // Already exported?
1542  if (FuncInfo.isExportedInst(V)) return;
1543 
1544  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1545  CopyValueToVirtualRegister(V, Reg);
1546 }
1547 
1549  const BasicBlock *FromBB) {
1550  // The operands of the setcc have to be in this block. We don't know
1551  // how to export them from some other block.
1552  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1553  // Can export from current BB.
1554  if (VI->getParent() == FromBB)
1555  return true;
1556 
1557  // Is already exported, noop.
1558  return FuncInfo.isExportedInst(V);
1559  }
1560 
1561  // If this is an argument, we can export it if the BB is the entry block or
1562  // if it is already exported.
1563  if (isa<Argument>(V)) {
1564  if (FromBB == &FromBB->getParent()->getEntryBlock())
1565  return true;
1566 
1567  // Otherwise, can only export this if it is already exported.
1568  return FuncInfo.isExportedInst(V);
1569  }
1570 
1571  // Otherwise, constants can always be exported.
1572  return true;
1573 }
1574 
1575 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1577 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1578  const MachineBasicBlock *Dst) const {
1579  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1580  const BasicBlock *SrcBB = Src->getBasicBlock();
1581  const BasicBlock *DstBB = Dst->getBasicBlock();
1582  if (!BPI) {
1583  // If BPI is not available, set the default probability as 1 / N, where N is
1584  // the number of successors.
1585  auto SuccSize = std::max<uint32_t>(
1586  std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1587  return BranchProbability(1, SuccSize);
1588  }
1589  return BPI->getEdgeProbability(SrcBB, DstBB);
1590 }
1591 
1592 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1593  MachineBasicBlock *Dst,
1594  BranchProbability Prob) {
1595  if (!FuncInfo.BPI)
1596  Src->addSuccessorWithoutProb(Dst);
1597  else {
1598  if (Prob.isUnknown())
1599  Prob = getEdgeProbability(Src, Dst);
1600  Src->addSuccessor(Dst, Prob);
1601  }
1602 }
1603 
1604 static bool InBlock(const Value *V, const BasicBlock *BB) {
1605  if (const Instruction *I = dyn_cast<Instruction>(V))
1606  return I->getParent() == BB;
1607  return true;
1608 }
1609 
1610 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1611 /// This function emits a branch and is used at the leaves of an OR or an
1612 /// AND operator tree.
1613 ///
1614 void
1616  MachineBasicBlock *TBB,
1617  MachineBasicBlock *FBB,
1618  MachineBasicBlock *CurBB,
1619  MachineBasicBlock *SwitchBB,
1620  BranchProbability TProb,
1621  BranchProbability FProb,
1622  bool InvertCond) {
1623  const BasicBlock *BB = CurBB->getBasicBlock();
1624 
1625  // If the leaf of the tree is a comparison, merge the condition into
1626  // the caseblock.
1627  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1628  // The operands of the cmp have to be in this block. We don't know
1629  // how to export them from some other block. If this is the first block
1630  // of the sequence, no exporting is needed.
1631  if (CurBB == SwitchBB ||
1632  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1633  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1634  ISD::CondCode Condition;
1635  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1636  ICmpInst::Predicate Pred =
1637  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1638  Condition = getICmpCondCode(Pred);
1639  } else {
1640  const FCmpInst *FC = cast<FCmpInst>(Cond);
1641  FCmpInst::Predicate Pred =
1642  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1643  Condition = getFCmpCondCode(Pred);
1644  if (TM.Options.NoNaNsFPMath)
1645  Condition = getFCmpCodeWithoutNaN(Condition);
1646  }
1647 
1648  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1649  TBB, FBB, CurBB, TProb, FProb);
1650  SwitchCases.push_back(CB);
1651  return;
1652  }
1653  }
1654 
1655  // Create a CaseBlock record representing this branch.
1656  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1657  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1658  nullptr, TBB, FBB, CurBB, TProb, FProb);
1659  SwitchCases.push_back(CB);
1660 }
1661 
1662 /// FindMergedConditions - If Cond is an expression like
1664  MachineBasicBlock *TBB,
1665  MachineBasicBlock *FBB,
1666  MachineBasicBlock *CurBB,
1667  MachineBasicBlock *SwitchBB,
1669  BranchProbability TProb,
1670  BranchProbability FProb,
1671  bool InvertCond) {
1672  // Skip over not part of the tree and remember to invert op and operands at
1673  // next level.
1674  if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1675  const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1676  if (InBlock(CondOp, CurBB->getBasicBlock())) {
1677  FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1678  !InvertCond);
1679  return;
1680  }
1681  }
1682 
1683  const Instruction *BOp = dyn_cast<Instruction>(Cond);
1684  // Compute the effective opcode for Cond, taking into account whether it needs
1685  // to be inverted, e.g.
1686  // and (not (or A, B)), C
1687  // gets lowered as
1688  // and (and (not A, not B), C)
1689  unsigned BOpc = 0;
1690  if (BOp) {
1691  BOpc = BOp->getOpcode();
1692  if (InvertCond) {
1693  if (BOpc == Instruction::And)
1694  BOpc = Instruction::Or;
1695  else if (BOpc == Instruction::Or)
1696  BOpc = Instruction::And;
1697  }
1698  }
1699 
1700  // If this node is not part of the or/and tree, emit it as a branch.
1701  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1702  BOpc != Opc || !BOp->hasOneUse() ||
1703  BOp->getParent() != CurBB->getBasicBlock() ||
1704  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1705  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1706  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1707  TProb, FProb, InvertCond);
1708  return;
1709  }
1710 
1711  // Create TmpBB after CurBB.
1712  MachineFunction::iterator BBI(CurBB);
1713  MachineFunction &MF = DAG.getMachineFunction();
1715  CurBB->getParent()->insert(++BBI, TmpBB);
1716 
1717  if (Opc == Instruction::Or) {
1718  // Codegen X | Y as:
1719  // BB1:
1720  // jmp_if_X TBB
1721  // jmp TmpBB
1722  // TmpBB:
1723  // jmp_if_Y TBB
1724  // jmp FBB
1725  //
1726 
1727  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1728  // The requirement is that
1729  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1730  // = TrueProb for original BB.
1731  // Assuming the original probabilities are A and B, one choice is to set
1732  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1733  // A/(1+B) and 2B/(1+B). This choice assumes that
1734  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1735  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1736  // TmpBB, but the math is more complicated.
1737 
1738  auto NewTrueProb = TProb / 2;
1739  auto NewFalseProb = TProb / 2 + FProb;
1740  // Emit the LHS condition.
1741  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1742  NewTrueProb, NewFalseProb, InvertCond);
1743 
1744  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1745  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1746  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1747  // Emit the RHS condition into TmpBB.
1748  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1749  Probs[0], Probs[1], InvertCond);
1750  } else {
1751  assert(Opc == Instruction::And && "Unknown merge op!");
1752  // Codegen X & Y as:
1753  // BB1:
1754  // jmp_if_X TmpBB
1755  // jmp FBB
1756  // TmpBB:
1757  // jmp_if_Y TBB
1758  // jmp FBB
1759  //
1760  // This requires creation of TmpBB after CurBB.
1761 
1762  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1763  // The requirement is that
1764  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1765  // = FalseProb for original BB.
1766  // Assuming the original probabilities are A and B, one choice is to set
1767  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1768  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1769  // TrueProb for BB1 * FalseProb for TmpBB.
1770 
1771  auto NewTrueProb = TProb + FProb / 2;
1772  auto NewFalseProb = FProb / 2;
1773  // Emit the LHS condition.
1774  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1775  NewTrueProb, NewFalseProb, InvertCond);
1776 
1777  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1778  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1779  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1780  // Emit the RHS condition into TmpBB.
1781  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1782  Probs[0], Probs[1], InvertCond);
1783  }
1784 }
1785 
1786 /// If the set of cases should be emitted as a series of branches, return true.
1787 /// If we should emit this as a bunch of and/or'd together conditions, return
1788 /// false.
1789 bool
1790 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1791  if (Cases.size() != 2) return true;
1792 
1793  // If this is two comparisons of the same values or'd or and'd together, they
1794  // will get folded into a single comparison, so don't emit two blocks.
1795  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1796  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1797  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1798  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1799  return false;
1800  }
1801 
1802  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1803  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1804  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1805  Cases[0].CC == Cases[1].CC &&
1806  isa<Constant>(Cases[0].CmpRHS) &&
1807  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1808  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1809  return false;
1810  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1811  return false;
1812  }
1813 
1814  return true;
1815 }
1816 
1817 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1818  MachineBasicBlock *BrMBB = FuncInfo.MBB;
1819 
1820  // Update machine-CFG edges.
1821  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1822 
1823  if (I.isUnconditional()) {
1824  // Update machine-CFG edges.
1825  BrMBB->addSuccessor(Succ0MBB);
1826 
1827  // If this is not a fall-through branch or optimizations are switched off,
1828  // emit the branch.
1829  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1830  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1831  MVT::Other, getControlRoot(),
1832  DAG.getBasicBlock(Succ0MBB)));
1833 
1834  return;
1835  }
1836 
1837  // If this condition is one of the special cases we handle, do special stuff
1838  // now.
1839  const Value *CondVal = I.getCondition();
1840  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1841 
1842  // If this is a series of conditions that are or'd or and'd together, emit
1843  // this as a sequence of branches instead of setcc's with and/or operations.
1844  // As long as jumps are not expensive, this should improve performance.
1845  // For example, instead of something like:
1846  // cmp A, B
1847  // C = seteq
1848  // cmp D, E
1849  // F = setle
1850  // or C, F
1851  // jnz foo
1852  // Emit:
1853  // cmp A, B
1854  // je foo
1855  // cmp D, E
1856  // jle foo
1857  //
1858  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1859  Instruction::BinaryOps Opcode = BOp->getOpcode();
1860  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1862  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1863  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1864  Opcode,
1865  getEdgeProbability(BrMBB, Succ0MBB),
1866  getEdgeProbability(BrMBB, Succ1MBB),
1867  /*InvertCond=*/false);
1868  // If the compares in later blocks need to use values not currently
1869  // exported from this block, export them now. This block should always
1870  // be the first entry.
1871  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
1872 
1873  // Allow some cases to be rejected.
1874  if (ShouldEmitAsBranches(SwitchCases)) {
1875  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1876  ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
1877  ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
1878  }
1879 
1880  // Emit the branch for this block.
1881  visitSwitchCase(SwitchCases[0], BrMBB);
1882  SwitchCases.erase(SwitchCases.begin());
1883  return;
1884  }
1885 
1886  // Okay, we decided not to do this, remove any inserted MBB's and clear
1887  // SwitchCases.
1888  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1889  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1890 
1891  SwitchCases.clear();
1892  }
1893  }
1894 
1895  // Create a CaseBlock record representing this branch.
1896  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1897  nullptr, Succ0MBB, Succ1MBB, BrMBB);
1898 
1899  // Use visitSwitchCase to actually insert the fast branch sequence for this
1900  // cond branch.
1901  visitSwitchCase(CB, BrMBB);
1902 }
1903 
1904 /// visitSwitchCase - Emits the necessary code to represent a single node in
1905 /// the binary search tree resulting from lowering a switch instruction.
1907  MachineBasicBlock *SwitchBB) {
1908  SDValue Cond;
1909  SDValue CondLHS = getValue(CB.CmpLHS);
1910  SDLoc dl = getCurSDLoc();
1911 
1912  // Build the setcc now.
1913  if (!CB.CmpMHS) {
1914  // Fold "(X == true)" to X and "(X == false)" to !X to
1915  // handle common cases produced by branch lowering.
1916  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1917  CB.CC == ISD::SETEQ)
1918  Cond = CondLHS;
1919  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1920  CB.CC == ISD::SETEQ) {
1921  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1922  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1923  } else
1924  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1925  } else {
1926  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
1927 
1928  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1929  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1930 
1931  SDValue CmpOp = getValue(CB.CmpMHS);
1932  EVT VT = CmpOp.getValueType();
1933 
1934  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1935  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1936  ISD::SETLE);
1937  } else {
1938  SDValue SUB = DAG.getNode(ISD::SUB, dl,
1939  VT, CmpOp, DAG.getConstant(Low, dl, VT));
1940  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1941  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1942  }
1943  }
1944 
1945  // Update successor info
1946  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1947  // TrueBB and FalseBB are always different unless the incoming IR is
1948  // degenerate. This only happens when running llc on weird IR.
1949  if (CB.TrueBB != CB.FalseBB)
1950  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1951  SwitchBB->normalizeSuccProbs();
1952 
1953  // If the lhs block is the next block, invert the condition so that we can
1954  // fall through to the lhs instead of the rhs block.
1955  if (CB.TrueBB == NextBlock(SwitchBB)) {
1956  std::swap(CB.TrueBB, CB.FalseBB);
1957  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1958  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1959  }
1960 
1961  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1962  MVT::Other, getControlRoot(), Cond,
1963  DAG.getBasicBlock(CB.TrueBB));
1964 
1965  // Insert the false branch. Do this even if it's a fall through branch,
1966  // this makes it easier to do DAG optimizations which require inverting
1967  // the branch condition.
1968  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1969  DAG.getBasicBlock(CB.FalseBB));
1970 
1971  DAG.setRoot(BrCond);
1972 }
1973 
1974 /// visitJumpTable - Emit JumpTable node in the current MBB
1976  // Emit the code for the jump table
1977  assert(JT.Reg != -1U && "Should lower JT Header first!");
1979  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
1980  JT.Reg, PTy);
1981  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1982  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1983  MVT::Other, Index.getValue(1),
1984  Table, Index);
1985  DAG.setRoot(BrJumpTable);
1986 }
1987 
1988 /// visitJumpTableHeader - This function emits necessary code to produce index
1989 /// in the JumpTable from switch case.
1991  JumpTableHeader &JTH,
1992  MachineBasicBlock *SwitchBB) {
1993  SDLoc dl = getCurSDLoc();
1994 
1995  // Subtract the lowest switch case value from the value being switched on and
1996  // conditional branch to default mbb if the result is greater than the
1997  // difference between smallest and largest cases.
1998  SDValue SwitchOp = getValue(JTH.SValue);
1999  EVT VT = SwitchOp.getValueType();
2000  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2001  DAG.getConstant(JTH.First, dl, VT));
2002 
2003  // The SDNode we just created, which holds the value being switched on minus
2004  // the smallest case value, needs to be copied to a virtual register so it
2005  // can be used as an index into the jump table in a subsequent basic block.
2006  // This value may be smaller or larger than the target's pointer type, and
2007  // therefore require extension or truncating.
2008  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2009  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2010 
2011  unsigned JumpTableReg =
2012  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2013  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2014  JumpTableReg, SwitchOp);
2015  JT.Reg = JumpTableReg;
2016 
2017  // Emit the range check for the jump table, and branch to the default block
2018  // for the switch statement if the value being switched on exceeds the largest
2019  // case in the switch.
2020  SDValue CMP = DAG.getSetCC(
2021  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2022  Sub.getValueType()),
2023  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2024 
2025  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2026  MVT::Other, CopyTo, CMP,
2027  DAG.getBasicBlock(JT.Default));
2028 
2029  // Avoid emitting unnecessary branches to the next block.
2030  if (JT.MBB != NextBlock(SwitchBB))
2031  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2032  DAG.getBasicBlock(JT.MBB));
2033 
2034  DAG.setRoot(BrCond);
2035 }
2036 
2037 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2038 /// variable if there exists one.
2040  SDValue &Chain) {
2041  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2042  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2043  MachineFunction &MF = DAG.getMachineFunction();
2044  Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent());
2045  MachineSDNode *Node =
2046  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2047  if (Global) {
2048  MachinePointerInfo MPInfo(Global);
2052  *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
2053  DAG.getEVTAlignment(PtrTy));
2054  Node->setMemRefs(MemRefs, MemRefs + 1);
2055  }
2056  return SDValue(Node, 0);
2057 }
2058 
2059 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2060 /// tail spliced into a stack protector check success bb.
2061 ///
2062 /// For a high level explanation of how this fits into the stack protector
2063 /// generation see the comment on the declaration of class
2064 /// StackProtectorDescriptor.
2065 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2066  MachineBasicBlock *ParentBB) {
2067 
2068  // First create the loads to the guard/stack slot for the comparison.
2069  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2070  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2071 
2072  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2073  int FI = MFI.getStackProtectorIndex();
2074 
2075  SDValue Guard;
2076  SDLoc dl = getCurSDLoc();
2077  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2078  const Module &M = *ParentBB->getParent()->getFunction()->getParent();
2079  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2080 
2081  // Generate code to load the content of the guard slot.
2082  SDValue StackSlot = DAG.getLoad(
2083  PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2086 
2087  // Retrieve guard check function, nullptr if instrumentation is inlined.
2088  if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2089  // The target provides a guard check function to validate the guard value.
2090  // Generate a call to that function with the content of the guard slot as
2091  // argument.
2092  auto *Fn = cast<Function>(GuardCheck);
2093  FunctionType *FnTy = Fn->getFunctionType();
2094  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2095 
2098  Entry.Node = StackSlot;
2099  Entry.Ty = FnTy->getParamType(0);
2100  if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2101  Entry.IsInReg = true;
2102  Args.push_back(Entry);
2103 
2105  CLI.setDebugLoc(getCurSDLoc())
2106  .setChain(DAG.getEntryNode())
2107  .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2108  getValue(GuardCheck), std::move(Args));
2109 
2110  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2111  DAG.setRoot(Result.second);
2112  return;
2113  }
2114 
2115  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2116  // Otherwise, emit a volatile load to retrieve the stack guard value.
2117  SDValue Chain = DAG.getEntryNode();
2118  if (TLI.useLoadStackGuardNode()) {
2119  Guard = getLoadStackGuard(DAG, dl, Chain);
2120  } else {
2121  const Value *IRGuard = TLI.getSDagStackGuard(M);
2122  SDValue GuardPtr = getValue(IRGuard);
2123 
2124  Guard =
2125  DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2127  }
2128 
2129  // Perform the comparison via a subtract/getsetcc.
2130  EVT VT = Guard.getValueType();
2131  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2132 
2133  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2134  *DAG.getContext(),
2135  Sub.getValueType()),
2136  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2137 
2138  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2139  // branch to failure MBB.
2140  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2141  MVT::Other, StackSlot.getOperand(0),
2142  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2143  // Otherwise branch to success MBB.
2144  SDValue Br = DAG.getNode(ISD::BR, dl,
2145  MVT::Other, BrCond,
2146  DAG.getBasicBlock(SPD.getSuccessMBB()));
2147 
2148  DAG.setRoot(Br);
2149 }
2150 
2151 /// Codegen the failure basic block for a stack protector check.
2152 ///
2153 /// A failure stack protector machine basic block consists simply of a call to
2154 /// __stack_chk_fail().
2155 ///
2156 /// For a high level explanation of how this fits into the stack protector
2157 /// generation see the comment on the declaration of class
2158 /// StackProtectorDescriptor.
2159 void
2160 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2161  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2162  SDValue Chain =
2163  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2164  None, false, getCurSDLoc(), false, false).second;
2165  DAG.setRoot(Chain);
2166 }
2167 
2168 /// visitBitTestHeader - This function emits necessary code to produce value
2169 /// suitable for "bit tests"
2171  MachineBasicBlock *SwitchBB) {
2172  SDLoc dl = getCurSDLoc();
2173 
2174  // Subtract the minimum value
2175  SDValue SwitchOp = getValue(B.SValue);
2176  EVT VT = SwitchOp.getValueType();
2177  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2178  DAG.getConstant(B.First, dl, VT));
2179 
2180  // Check range
2181  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2182  SDValue RangeCmp = DAG.getSetCC(
2183  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2184  Sub.getValueType()),
2185  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2186 
2187  // Determine the type of the test operands.
2188  bool UsePtrType = false;
2189  if (!TLI.isTypeLegal(VT))
2190  UsePtrType = true;
2191  else {
2192  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2193  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2194  // Switch table case range are encoded into series of masks.
2195  // Just use pointer type, it's guaranteed to fit.
2196  UsePtrType = true;
2197  break;
2198  }
2199  }
2200  if (UsePtrType) {
2201  VT = TLI.getPointerTy(DAG.getDataLayout());
2202  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2203  }
2204 
2205  B.RegVT = VT.getSimpleVT();
2206  B.Reg = FuncInfo.CreateReg(B.RegVT);
2207  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2208 
2209  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2210 
2211  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2212  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2213  SwitchBB->normalizeSuccProbs();
2214 
2215  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2216  MVT::Other, CopyTo, RangeCmp,
2217  DAG.getBasicBlock(B.Default));
2218 
2219  // Avoid emitting unnecessary branches to the next block.
2220  if (MBB != NextBlock(SwitchBB))
2221  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2222  DAG.getBasicBlock(MBB));
2223 
2224  DAG.setRoot(BrRange);
2225 }
2226 
2227 /// visitBitTestCase - this function produces one "bit test"
2229  MachineBasicBlock* NextMBB,
2230  BranchProbability BranchProbToNext,
2231  unsigned Reg,
2232  BitTestCase &B,
2233  MachineBasicBlock *SwitchBB) {
2234  SDLoc dl = getCurSDLoc();
2235  MVT VT = BB.RegVT;
2236  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2237  SDValue Cmp;
2238  unsigned PopCount = countPopulation(B.Mask);
2239  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2240  if (PopCount == 1) {
2241  // Testing for a single bit; just compare the shift count with what it
2242  // would need to be to shift a 1 bit in that position.
2243  Cmp = DAG.getSetCC(
2244  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2245  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2246  ISD::SETEQ);
2247  } else if (PopCount == BB.Range) {
2248  // There is only one zero bit in the range, test for it directly.
2249  Cmp = DAG.getSetCC(
2250  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2251  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2252  ISD::SETNE);
2253  } else {
2254  // Make desired shift
2255  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2256  DAG.getConstant(1, dl, VT), ShiftOp);
2257 
2258  // Emit bit tests and jumps
2259  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2260  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2261  Cmp = DAG.getSetCC(
2262  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2263  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2264  }
2265 
2266  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2267  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2268  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2269  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2270  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2271  // one as they are relative probabilities (and thus work more like weights),
2272  // and hence we need to normalize them to let the sum of them become one.
2273  SwitchBB->normalizeSuccProbs();
2274 
2275  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2276  MVT::Other, getControlRoot(),
2277  Cmp, DAG.getBasicBlock(B.TargetBB));
2278 
2279  // Avoid emitting unnecessary branches to the next block.
2280  if (NextMBB != NextBlock(SwitchBB))
2281  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2282  DAG.getBasicBlock(NextMBB));
2283 
2284  DAG.setRoot(BrAnd);
2285 }
2286 
2287 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2288  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2289 
2290  // Retrieve successors. Look through artificial IR level blocks like
2291  // catchswitch for successors.
2292  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2293  const BasicBlock *EHPadBB = I.getSuccessor(1);
2294 
2295  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2296  // have to do anything here to lower funclet bundles.
2298  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2299  "Cannot lower invokes with arbitrary operand bundles yet!");
2300 
2301  const Value *Callee(I.getCalledValue());
2302  const Function *Fn = dyn_cast<Function>(Callee);
2303  if (isa<InlineAsm>(Callee))
2304  visitInlineAsm(&I);
2305  else if (Fn && Fn->isIntrinsic()) {
2306  switch (Fn->getIntrinsicID()) {
2307  default:
2308  llvm_unreachable("Cannot invoke this intrinsic");
2309  case Intrinsic::donothing:
2310  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2311  break;
2312  case Intrinsic::experimental_patchpoint_void:
2313  case Intrinsic::experimental_patchpoint_i64:
2314  visitPatchpoint(&I, EHPadBB);
2315  break;
2316  case Intrinsic::experimental_gc_statepoint:
2317  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2318  break;
2319  }
2321  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2322  // Eventually we will support lowering the @llvm.experimental.deoptimize
2323  // intrinsic, and right now there are no plans to support other intrinsics
2324  // with deopt state.
2325  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2326  } else {
2327  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2328  }
2329 
2330  // If the value of the invoke is used outside of its defining block, make it
2331  // available as a virtual register.
2332  // We already took care of the exported value for the statepoint instruction
2333  // during call to the LowerStatepoint.
2334  if (!isStatepoint(I)) {
2335  CopyToExportRegsIfNeeded(&I);
2336  }
2337 
2339  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2340  BranchProbability EHPadBBProb =
2341  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2343  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2344 
2345  // Update successor info.
2346  addSuccessorWithProb(InvokeMBB, Return);
2347  for (auto &UnwindDest : UnwindDests) {
2348  UnwindDest.first->setIsEHPad();
2349  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2350  }
2351  InvokeMBB->normalizeSuccProbs();
2352 
2353  // Drop into normal successor.
2354  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2355  MVT::Other, getControlRoot(),
2356  DAG.getBasicBlock(Return)));
2357 }
2358 
2359 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2360  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2361 }
2362 
2363 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2364  assert(FuncInfo.MBB->isEHPad() &&
2365  "Call to landingpad not in landing pad!");
2366 
2367  MachineBasicBlock *MBB = FuncInfo.MBB;
2368  addLandingPadInfo(LP, *MBB);
2369 
2370  // If there aren't registers to copy the values into (e.g., during SjLj
2371  // exceptions), then don't bother to create these DAG nodes.
2372  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2373  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2374  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2375  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2376  return;
2377 
2378  // If landingpad's return type is token type, we don't create DAG nodes
2379  // for its exception pointer and selector value. The extraction of exception
2380  // pointer or selector value from token type landingpads is not currently
2381  // supported.
2382  if (LP.getType()->isTokenTy())
2383  return;
2384 
2386  SDLoc dl = getCurSDLoc();
2387  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2388  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2389 
2390  // Get the two live-in registers as SDValues. The physregs have already been
2391  // copied into virtual registers.
2392  SDValue Ops[2];
2393  if (FuncInfo.ExceptionPointerVirtReg) {
2394  Ops[0] = DAG.getZExtOrTrunc(
2395  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2396  FuncInfo.ExceptionPointerVirtReg,
2397  TLI.getPointerTy(DAG.getDataLayout())),
2398  dl, ValueVTs[0]);
2399  } else {
2400  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2401  }
2402  Ops[1] = DAG.getZExtOrTrunc(
2403  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2404  FuncInfo.ExceptionSelectorVirtReg,
2405  TLI.getPointerTy(DAG.getDataLayout())),
2406  dl, ValueVTs[1]);
2407 
2408  // Merge into one.
2409  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2410  DAG.getVTList(ValueVTs), Ops);
2411  setValue(&LP, Res);
2412 }
2413 
2414 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2415 #ifndef NDEBUG
2416  for (const CaseCluster &CC : Clusters)
2417  assert(CC.Low == CC.High && "Input clusters must be single-case");
2418 #endif
2419 
2420  std::sort(Clusters.begin(), Clusters.end(),
2421  [](const CaseCluster &a, const CaseCluster &b) {
2422  return a.Low->getValue().slt(b.Low->getValue());
2423  });
2424 
2425  // Merge adjacent clusters with the same destination.
2426  const unsigned N = Clusters.size();
2427  unsigned DstIndex = 0;
2428  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2429  CaseCluster &CC = Clusters[SrcIndex];
2430  const ConstantInt *CaseVal = CC.Low;
2431  MachineBasicBlock *Succ = CC.MBB;
2432 
2433  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2434  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2435  // If this case has the same successor and is a neighbour, merge it into
2436  // the previous cluster.
2437  Clusters[DstIndex - 1].High = CaseVal;
2438  Clusters[DstIndex - 1].Prob += CC.Prob;
2439  } else {
2440  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2441  sizeof(Clusters[SrcIndex]));
2442  }
2443  }
2444  Clusters.resize(DstIndex);
2445 }
2446 
2448  MachineBasicBlock *Last) {
2449  // Update JTCases.
2450  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2451  if (JTCases[i].first.HeaderBB == First)
2452  JTCases[i].first.HeaderBB = Last;
2453 
2454  // Update BitTestCases.
2455  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2456  if (BitTestCases[i].Parent == First)
2457  BitTestCases[i].Parent = Last;
2458 }
2459 
2460 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2461  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2462 
2463  // Update machine-CFG edges with unique successors.
2465  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2466  BasicBlock *BB = I.getSuccessor(i);
2467  bool Inserted = Done.insert(BB).second;
2468  if (!Inserted)
2469  continue;
2470 
2471  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2472  addSuccessorWithProb(IndirectBrMBB, Succ);
2473  }
2474  IndirectBrMBB->normalizeSuccProbs();
2475 
2476  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2477  MVT::Other, getControlRoot(),
2478  getValue(I.getAddress())));
2479 }
2480 
2481 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2482  if (DAG.getTarget().Options.TrapUnreachable)
2483  DAG.setRoot(
2484  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2485 }
2486 
2487 void SelectionDAGBuilder::visitFSub(const User &I) {
2488  // -0.0 - X --> fneg
2489  Type *Ty = I.getType();
2490  if (isa<Constant>(I.getOperand(0)) &&
2492  SDValue Op2 = getValue(I.getOperand(1));
2493  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2494  Op2.getValueType(), Op2));
2495  return;
2496  }
2497 
2498  visitBinary(I, ISD::FSUB);
2499 }
2500 
2501 /// Checks if the given instruction performs a vector reduction, in which case
2502 /// we have the freedom to alter the elements in the result as long as the
2503 /// reduction of them stays unchanged.
2504 static bool isVectorReductionOp(const User *I) {
2505  const Instruction *Inst = dyn_cast<Instruction>(I);
2506  if (!Inst || !Inst->getType()->isVectorTy())
2507  return false;
2508 
2509  auto OpCode = Inst->getOpcode();
2510  switch (OpCode) {
2511  case Instruction::Add:
2512  case Instruction::Mul:
2513  case Instruction::And:
2514  case Instruction::Or:
2515  case Instruction::Xor:
2516  break;
2517  case Instruction::FAdd:
2518  case Instruction::FMul:
2519  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2520  if (FPOp->getFastMathFlags().unsafeAlgebra())
2521  break;
2523  default:
2524  return false;
2525  }
2526 
2527  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2528  unsigned ElemNumToReduce = ElemNum;
2529 
2530  // Do DFS search on the def-use chain from the given instruction. We only
2531  // allow four kinds of operations during the search until we reach the
2532  // instruction that extracts the first element from the vector:
2533  //
2534  // 1. The reduction operation of the same opcode as the given instruction.
2535  //
2536  // 2. PHI node.
2537  //
2538  // 3. ShuffleVector instruction together with a reduction operation that
2539  // does a partial reduction.
2540  //
2541  // 4. ExtractElement that extracts the first element from the vector, and we
2542  // stop searching the def-use chain here.
2543  //
2544  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2545  // from 1-3 to the stack to continue the DFS. The given instruction is not
2546  // a reduction operation if we meet any other instructions other than those
2547  // listed above.
2548 
2549  SmallVector<const User *, 16> UsersToVisit{Inst};
2551  bool ReduxExtracted = false;
2552 
2553  while (!UsersToVisit.empty()) {
2554  auto User = UsersToVisit.back();
2555  UsersToVisit.pop_back();
2556  if (!Visited.insert(User).second)
2557  continue;
2558 
2559  for (const auto &U : User->users()) {
2560  auto Inst = dyn_cast<Instruction>(U);
2561  if (!Inst)
2562  return false;
2563 
2564  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2565  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2566  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2567  return false;
2568  UsersToVisit.push_back(U);
2569  } else if (const ShuffleVectorInst *ShufInst =
2570  dyn_cast<ShuffleVectorInst>(U)) {
2571  // Detect the following pattern: A ShuffleVector instruction together
2572  // with a reduction that do partial reduction on the first and second
2573  // ElemNumToReduce / 2 elements, and store the result in
2574  // ElemNumToReduce / 2 elements in another vector.
2575 
2576  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2577  if (ResultElements < ElemNum)
2578  return false;
2579 
2580  if (ElemNumToReduce == 1)
2581  return false;
2582  if (!isa<UndefValue>(U->getOperand(1)))
2583  return false;
2584  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2585  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2586  return false;
2587  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2588  if (ShufInst->getMaskValue(i) != -1)
2589  return false;
2590 
2591  // There is only one user of this ShuffleVector instruction, which
2592  // must be a reduction operation.
2593  if (!U->hasOneUse())
2594  return false;
2595 
2596  auto U2 = dyn_cast<Instruction>(*U->user_begin());
2597  if (!U2 || U2->getOpcode() != OpCode)
2598  return false;
2599 
2600  // Check operands of the reduction operation.
2601  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2602  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2603  UsersToVisit.push_back(U2);
2604  ElemNumToReduce /= 2;
2605  } else
2606  return false;
2607  } else if (isa<ExtractElementInst>(U)) {
2608  // At this moment we should have reduced all elements in the vector.
2609  if (ElemNumToReduce != 1)
2610  return false;
2611 
2612  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2613  if (!Val || Val->getZExtValue() != 0)
2614  return false;
2615 
2616  ReduxExtracted = true;
2617  } else
2618  return false;
2619  }
2620  }
2621  return ReduxExtracted;
2622 }
2623 
2624 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2625  SDValue Op1 = getValue(I.getOperand(0));
2626  SDValue Op2 = getValue(I.getOperand(1));
2627 
2628  bool nuw = false;
2629  bool nsw = false;
2630  bool exact = false;
2631  bool vec_redux = false;
2632  FastMathFlags FMF;
2633 
2634  if (const OverflowingBinaryOperator *OFBinOp =
2635  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2636  nuw = OFBinOp->hasNoUnsignedWrap();
2637  nsw = OFBinOp->hasNoSignedWrap();
2638  }
2639  if (const PossiblyExactOperator *ExactOp =
2640  dyn_cast<const PossiblyExactOperator>(&I))
2641  exact = ExactOp->isExact();
2642  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2643  FMF = FPOp->getFastMathFlags();
2644 
2645  if (isVectorReductionOp(&I)) {
2646  vec_redux = true;
2647  DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2648  }
2649 
2651  Flags.setExact(exact);
2652  Flags.setNoSignedWrap(nsw);
2653  Flags.setNoUnsignedWrap(nuw);
2654  Flags.setVectorReduction(vec_redux);
2655  Flags.setAllowReciprocal(FMF.allowReciprocal());
2656  Flags.setAllowContract(FMF.allowContract());
2657  Flags.setNoInfs(FMF.noInfs());
2658  Flags.setNoNaNs(FMF.noNaNs());
2659  Flags.setNoSignedZeros(FMF.noSignedZeros());
2660  Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2661 
2662  SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2663  Op1, Op2, Flags);
2664  setValue(&I, BinNodeValue);
2665 }
2666 
2667 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2668  SDValue Op1 = getValue(I.getOperand(0));
2669  SDValue Op2 = getValue(I.getOperand(1));
2670 
2671  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2672  Op2.getValueType(), DAG.getDataLayout());
2673 
2674  // Coerce the shift amount to the right type if we can.
2675  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2676  unsigned ShiftSize = ShiftTy.getSizeInBits();
2677  unsigned Op2Size = Op2.getValueSizeInBits();
2678  SDLoc DL = getCurSDLoc();
2679 
2680  // If the operand is smaller than the shift count type, promote it.
2681  if (ShiftSize > Op2Size)
2682  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2683 
2684  // If the operand is larger than the shift count type but the shift
2685  // count type has enough bits to represent any shift value, truncate
2686  // it now. This is a common case and it exposes the truncate to
2687  // optimization early.
2688  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2689  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2690  // Otherwise we'll need to temporarily settle for some other convenient
2691  // type. Type legalization will make adjustments once the shiftee is split.
2692  else
2693  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2694  }
2695 
2696  bool nuw = false;
2697  bool nsw = false;
2698  bool exact = false;
2699 
2700  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2701 
2702  if (const OverflowingBinaryOperator *OFBinOp =
2703  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2704  nuw = OFBinOp->hasNoUnsignedWrap();
2705  nsw = OFBinOp->hasNoSignedWrap();
2706  }
2707  if (const PossiblyExactOperator *ExactOp =
2708  dyn_cast<const PossiblyExactOperator>(&I))
2709  exact = ExactOp->isExact();
2710  }
2712  Flags.setExact(exact);
2713  Flags.setNoSignedWrap(nsw);
2714  Flags.setNoUnsignedWrap(nuw);
2715  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2716  Flags);
2717  setValue(&I, Res);
2718 }
2719 
2720 void SelectionDAGBuilder::visitSDiv(const User &I) {
2721  SDValue Op1 = getValue(I.getOperand(0));
2722  SDValue Op2 = getValue(I.getOperand(1));
2723 
2725  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2726  cast<PossiblyExactOperator>(&I)->isExact());
2727  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2728  Op2, Flags));
2729 }
2730 
2731 void SelectionDAGBuilder::visitICmp(const User &I) {
2733  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2734  predicate = IC->getPredicate();
2735  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2736  predicate = ICmpInst::Predicate(IC->getPredicate());
2737  SDValue Op1 = getValue(I.getOperand(0));
2738  SDValue Op2 = getValue(I.getOperand(1));
2739  ISD::CondCode Opcode = getICmpCondCode(predicate);
2740 
2741  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2742  I.getType());
2743  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2744 }
2745 
2746 void SelectionDAGBuilder::visitFCmp(const User &I) {
2748  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2749  predicate = FC->getPredicate();
2750  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2751  predicate = FCmpInst::Predicate(FC->getPredicate());
2752  SDValue Op1 = getValue(I.getOperand(0));
2753  SDValue Op2 = getValue(I.getOperand(1));
2754  ISD::CondCode Condition = getFCmpCondCode(predicate);
2755 
2756  // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2757  // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2758  // further optimization, but currently FMF is only applicable to binary nodes.
2759  if (TM.Options.NoNaNsFPMath)
2760  Condition = getFCmpCodeWithoutNaN(Condition);
2761  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2762  I.getType());
2763  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2764 }
2765 
2766 // Check if the condition of the select has one use or two users that are both
2767 // selects with the same condition.
2768 static bool hasOnlySelectUsers(const Value *Cond) {
2769  return all_of(Cond->users(), [](const Value *V) {
2770  return isa<SelectInst>(V);
2771  });
2772 }
2773 
2774 void SelectionDAGBuilder::visitSelect(const User &I) {
2777  ValueVTs);
2778  unsigned NumValues = ValueVTs.size();
2779  if (NumValues == 0) return;
2780 
2781  SmallVector<SDValue, 4> Values(NumValues);
2782  SDValue Cond = getValue(I.getOperand(0));
2783  SDValue LHSVal = getValue(I.getOperand(1));
2784  SDValue RHSVal = getValue(I.getOperand(2));
2785  auto BaseOps = {Cond};
2786  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2788 
2789  // Min/max matching is only viable if all output VTs are the same.
2790  if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2791  EVT VT = ValueVTs[0];
2792  LLVMContext &Ctx = *DAG.getContext();
2793  auto &TLI = DAG.getTargetLoweringInfo();
2794 
2795  // We care about the legality of the operation after it has been type
2796  // legalized.
2797  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2798  VT != TLI.getTypeToTransformTo(Ctx, VT))
2799  VT = TLI.getTypeToTransformTo(Ctx, VT);
2800 
2801  // If the vselect is legal, assume we want to leave this as a vector setcc +
2802  // vselect. Otherwise, if this is going to be scalarized, we want to see if
2803  // min/max is legal on the scalar type.
2804  bool UseScalarMinMax = VT.isVector() &&
2805  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2806 
2807  Value *LHS, *RHS;
2808  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2810  switch (SPR.Flavor) {
2811  case SPF_UMAX: Opc = ISD::UMAX; break;
2812  case SPF_UMIN: Opc = ISD::UMIN; break;
2813  case SPF_SMAX: Opc = ISD::SMAX; break;
2814  case SPF_SMIN: Opc = ISD::SMIN; break;
2815  case SPF_FMINNUM:
2816  switch (SPR.NaNBehavior) {
2817  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2818  case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2819  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2820  case SPNB_RETURNS_ANY: {
2821  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2822  Opc = ISD::FMINNUM;
2823  else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2824  Opc = ISD::FMINNAN;
2825  else if (UseScalarMinMax)
2826  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2828  break;
2829  }
2830  }
2831  break;
2832  case SPF_FMAXNUM:
2833  switch (SPR.NaNBehavior) {
2834  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2835  case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2836  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2837  case SPNB_RETURNS_ANY:
2838 
2839  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2840  Opc = ISD::FMAXNUM;
2841  else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2842  Opc = ISD::FMAXNAN;
2843  else if (UseScalarMinMax)
2844  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2846  break;
2847  }
2848  break;
2849  default: break;
2850  }
2851 
2852  if (Opc != ISD::DELETED_NODE &&
2853  (TLI.isOperationLegalOrCustom(Opc, VT) ||
2854  (UseScalarMinMax &&
2855  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2856  // If the underlying comparison instruction is used by any other
2857  // instruction, the consumed instructions won't be destroyed, so it is
2858  // not profitable to convert to a min/max.
2859  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2860  OpCode = Opc;
2861  LHSVal = getValue(LHS);
2862  RHSVal = getValue(RHS);
2863  BaseOps = {};
2864  }
2865  }
2866 
2867  for (unsigned i = 0; i != NumValues; ++i) {
2868  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2869  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2870  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2871  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2872  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2873  Ops);
2874  }
2875 
2876  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
2877  DAG.getVTList(ValueVTs), Values));
2878 }
2879 
2880 void SelectionDAGBuilder::visitTrunc(const User &I) {
2881  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2882  SDValue N = getValue(I.getOperand(0));
2883  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2884  I.getType());
2885  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2886 }
2887 
2888 void SelectionDAGBuilder::visitZExt(const User &I) {
2889  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2890  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2891  SDValue N = getValue(I.getOperand(0));
2892  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2893  I.getType());
2894  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2895 }
2896 
2897 void SelectionDAGBuilder::visitSExt(const User &I) {
2898  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2899  // SExt also can't be a cast to bool for same reason. So, nothing much to do
2900  SDValue N = getValue(I.getOperand(0));
2901  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2902  I.getType());
2903  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2904 }
2905 
2906 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2907  // FPTrunc is never a no-op cast, no need to check
2908  SDValue N = getValue(I.getOperand(0));
2909  SDLoc dl = getCurSDLoc();
2910  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2911  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2912  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2913  DAG.getTargetConstant(
2914  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2915 }
2916 
2917 void SelectionDAGBuilder::visitFPExt(const User &I) {
2918  // FPExt is never a no-op cast, no need to check
2919  SDValue N = getValue(I.getOperand(0));
2920  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2921  I.getType());
2922  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2923 }
2924 
2925 void SelectionDAGBuilder::visitFPToUI(const User &I) {
2926  // FPToUI is never a no-op cast, no need to check
2927  SDValue N = getValue(I.getOperand(0));
2928  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2929  I.getType());
2930  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2931 }
2932 
2933 void SelectionDAGBuilder::visitFPToSI(const User &I) {
2934  // FPToSI is never a no-op cast, no need to check
2935  SDValue N = getValue(I.getOperand(0));
2936  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2937  I.getType());
2938  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2939 }
2940 
2941 void SelectionDAGBuilder::visitUIToFP(const User &I) {
2942  // UIToFP is never a no-op cast, no need to check
2943  SDValue N = getValue(I.getOperand(0));
2944  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2945  I.getType());
2946  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2947 }
2948 
2949 void SelectionDAGBuilder::visitSIToFP(const User &I) {
2950  // SIToFP is never a no-op cast, no need to check
2951  SDValue N = getValue(I.getOperand(0));
2952  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2953  I.getType());
2954  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2955 }
2956 
2957 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2958  // What to do depends on the size of the integer and the size of the pointer.
2959  // We can either truncate, zero extend, or no-op, accordingly.
2960  SDValue N = getValue(I.getOperand(0));
2961  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2962  I.getType());
2963  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2964 }
2965 
2966 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2967  // What to do depends on the size of the integer and the size of the pointer.
2968  // We can either truncate, zero extend, or no-op, accordingly.
2969  SDValue N = getValue(I.getOperand(0));
2970  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2971  I.getType());
2972  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2973 }
2974 
2975 void SelectionDAGBuilder::visitBitCast(const User &I) {
2976  SDValue N = getValue(I.getOperand(0));
2977  SDLoc dl = getCurSDLoc();
2978  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2979  I.getType());
2980 
2981  // BitCast assures us that source and destination are the same size so this is
2982  // either a BITCAST or a no-op.
2983  if (DestVT != N.getValueType())
2984  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2985  DestVT, N)); // convert types.
2986  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2987  // might fold any kind of constant expression to an integer constant and that
2988  // is not what we are looking for. Only recognize a bitcast of a genuine
2989  // constant integer as an opaque constant.
2990  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2991  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2992  /*isOpaque*/true));
2993  else
2994  setValue(&I, N); // noop cast.
2995 }
2996 
2997 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2998  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2999  const Value *SV = I.getOperand(0);
3000  SDValue N = getValue(SV);
3001  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3002 
3003  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3004  unsigned DestAS = I.getType()->getPointerAddressSpace();
3005 
3006  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3007  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3008 
3009  setValue(&I, N);
3010 }
3011 
3012 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3013  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3014  SDValue InVec = getValue(I.getOperand(0));
3015  SDValue InVal = getValue(I.getOperand(1));
3016  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3017  TLI.getVectorIdxTy(DAG.getDataLayout()));
3018  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3019  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3020  InVec, InVal, InIdx));
3021 }
3022 
3023 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3024  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3025  SDValue InVec = getValue(I.getOperand(0));
3026  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3027  TLI.getVectorIdxTy(DAG.getDataLayout()));
3028  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3029  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3030  InVec, InIdx));
3031 }
3032 
3033 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3034  SDValue Src1 = getValue(I.getOperand(0));
3035  SDValue Src2 = getValue(I.getOperand(1));
3036  SDLoc DL = getCurSDLoc();
3037 
3039  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3040  unsigned MaskNumElts = Mask.size();
3041 
3042  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3043  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3044  EVT SrcVT = Src1.getValueType();
3045  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3046 
3047  if (SrcNumElts == MaskNumElts) {
3048  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3049  return;
3050  }
3051 
3052  // Normalize the shuffle vector since mask and vector length don't match.
3053  if (SrcNumElts < MaskNumElts) {
3054  // Mask is longer than the source vectors. We can use concatenate vector to
3055  // make the mask and vectors lengths match.
3056 
3057  if (MaskNumElts % SrcNumElts == 0) {
3058  // Mask length is a multiple of the source vector length.
3059  // Check if the shuffle is some kind of concatenation of the input
3060  // vectors.
3061  unsigned NumConcat = MaskNumElts / SrcNumElts;
3062  bool IsConcat = true;
3063  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3064  for (unsigned i = 0; i != MaskNumElts; ++i) {
3065  int Idx = Mask[i];
3066  if (Idx < 0)
3067  continue;
3068  // Ensure the indices in each SrcVT sized piece are sequential and that
3069  // the same source is used for the whole piece.
3070  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3071  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3072  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3073  IsConcat = false;
3074  break;
3075  }
3076  // Remember which source this index came from.
3077  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3078  }
3079 
3080  // The shuffle is concatenating multiple vectors together. Just emit
3081  // a CONCAT_VECTORS operation.
3082  if (IsConcat) {
3083  SmallVector<SDValue, 8> ConcatOps;
3084  for (auto Src : ConcatSrcs) {
3085  if (Src < 0)
3086  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3087  else if (Src == 0)
3088  ConcatOps.push_back(Src1);
3089  else
3090  ConcatOps.push_back(Src2);
3091  }
3092  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3093  return;
3094  }
3095  }
3096 
3097  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3098  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3099  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3100  PaddedMaskNumElts);
3101 
3102  // Pad both vectors with undefs to make them the same length as the mask.
3103  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3104 
3105  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3106  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3107  MOps1[0] = Src1;
3108  MOps2[0] = Src2;
3109 
3110  Src1 = Src1.isUndef()
3111  ? DAG.getUNDEF(PaddedVT)
3112  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3113  Src2 = Src2.isUndef()
3114  ? DAG.getUNDEF(PaddedVT)
3115  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3116 
3117  // Readjust mask for new input vector length.
3118  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3119  for (unsigned i = 0; i != MaskNumElts; ++i) {
3120  int Idx = Mask[i];
3121  if (Idx >= (int)SrcNumElts)
3122  Idx -= SrcNumElts - PaddedMaskNumElts;
3123  MappedOps[i] = Idx;
3124  }
3125 
3126  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3127 
3128  // If the concatenated vector was padded, extract a subvector with the
3129  // correct number of elements.
3130  if (MaskNumElts != PaddedMaskNumElts)
3131  Result = DAG.getNode(
3132  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3133  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3134 
3135  setValue(&I, Result);
3136  return;
3137  }
3138 
3139  if (SrcNumElts > MaskNumElts) {
3140  // Analyze the access pattern of the vector to see if we can extract
3141  // two subvectors and do the shuffle.
3142  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3143  bool CanExtract = true;
3144  for (int Idx : Mask) {
3145  unsigned Input = 0;
3146  if (Idx < 0)
3147  continue;
3148 
3149  if (Idx >= (int)SrcNumElts) {
3150  Input = 1;
3151  Idx -= SrcNumElts;
3152  }
3153 
3154  // If all the indices come from the same MaskNumElts sized portion of
3155  // the sources we can use extract. Also make sure the extract wouldn't
3156  // extract past the end of the source.
3157  int NewStartIdx = alignDown(Idx, MaskNumElts);
3158  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3159  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3160  CanExtract = false;
3161  // Make sure we always update StartIdx as we use it to track if all
3162  // elements are undef.
3163  StartIdx[Input] = NewStartIdx;
3164  }
3165 
3166  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3167  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3168  return;
3169  }
3170  if (CanExtract) {
3171  // Extract appropriate subvector and generate a vector shuffle
3172  for (unsigned Input = 0; Input < 2; ++Input) {
3173  SDValue &Src = Input == 0 ? Src1 : Src2;
3174  if (StartIdx[Input] < 0)
3175  Src = DAG.getUNDEF(VT);
3176  else {
3177  Src = DAG.getNode(
3178  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3179  DAG.getConstant(StartIdx[Input], DL,
3180  TLI.getVectorIdxTy(DAG.getDataLayout())));
3181  }
3182  }
3183 
3184  // Calculate new mask.
3185  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3186  for (int &Idx : MappedOps) {
3187  if (Idx >= (int)SrcNumElts)
3188  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3189  else if (Idx >= 0)
3190  Idx -= StartIdx[0];
3191  }
3192 
3193  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3194  return;
3195  }
3196  }
3197 
3198  // We can't use either concat vectors or extract subvectors so fall back to
3199  // replacing the shuffle with extract and build vector.
3200  // to insert and build vector.
3201  EVT EltVT = VT.getVectorElementType();
3202  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3204  for (int Idx : Mask) {
3205  SDValue Res;
3206 
3207  if (Idx < 0) {
3208  Res = DAG.getUNDEF(EltVT);
3209  } else {
3210  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3211  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3212 
3213  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3214  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3215  }
3216 
3217  Ops.push_back(Res);
3218  }
3219 
3220  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3221 }
3222 
3223 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3224  ArrayRef<unsigned> Indices;
3225  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3226  Indices = IV->getIndices();
3227  else
3228  Indices = cast<ConstantExpr>(&I)->getIndices();
3229 
3230  const Value *Op0 = I.getOperand(0);
3231  const Value *Op1 = I.getOperand(1);
3232  Type *AggTy = I.getType();
3233  Type *ValTy = Op1->getType();
3234  bool IntoUndef = isa<UndefValue>(Op0);
3235  bool FromUndef = isa<UndefValue>(Op1);
3236 
3237  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3238 
3239  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3240  SmallVector<EVT, 4> AggValueVTs;
3241  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3242  SmallVector<EVT, 4> ValValueVTs;
3243  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3244 
3245  unsigned NumAggValues = AggValueVTs.size();
3246  unsigned NumValValues = ValValueVTs.size();
3247  SmallVector<SDValue, 4> Values(NumAggValues);
3248 
3249  // Ignore an insertvalue that produces an empty object
3250  if (!NumAggValues) {
3251  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3252  return;
3253  }
3254 
3255  SDValue Agg = getValue(Op0);
3256  unsigned i = 0;
3257  // Copy the beginning value(s) from the original aggregate.
3258  for (; i != LinearIndex; ++i)
3259  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3260  SDValue(Agg.getNode(), Agg.getResNo() + i);
3261  // Copy values from the inserted value(s).
3262  if (NumValValues) {
3263  SDValue Val = getValue(Op1);
3264  for (; i != LinearIndex + NumValValues; ++i)
3265  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3266  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3267  }
3268  // Copy remaining value(s) from the original aggregate.
3269  for (; i != NumAggValues; ++i)
3270  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3271  SDValue(Agg.getNode(), Agg.getResNo() + i);
3272 
3273  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3274  DAG.getVTList(AggValueVTs), Values));
3275 }
3276 
3277 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3278  ArrayRef<unsigned> Indices;
3279  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3280  Indices = EV->getIndices();
3281  else
3282  Indices = cast<ConstantExpr>(&I)->getIndices();
3283 
3284  const Value *Op0 = I.getOperand(0);
3285  Type *AggTy = Op0->getType();
3286  Type *ValTy = I.getType();
3287  bool OutOfUndef = isa<UndefValue>(Op0);
3288 
3289  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3290 
3291  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3292  SmallVector<EVT, 4> ValValueVTs;
3293  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3294 
3295  unsigned NumValValues = ValValueVTs.size();
3296 
3297  // Ignore a extractvalue that produces an empty object
3298  if (!NumValValues) {
3299  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3300  return;
3301  }
3302 
3303  SmallVector<SDValue, 4> Values(NumValValues);
3304 
3305  SDValue Agg = getValue(Op0);
3306  // Copy out the selected value(s).
3307  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3308  Values[i - LinearIndex] =
3309  OutOfUndef ?
3310  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3311  SDValue(Agg.getNode(), Agg.getResNo() + i);
3312 
3313  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3314  DAG.getVTList(ValValueVTs), Values));
3315 }
3316 
3317 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3318  Value *Op0 = I.getOperand(0);
3319  // Note that the pointer operand may be a vector of pointers. Take the scalar
3320  // element which holds a pointer.
3321  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3322  SDValue N = getValue(Op0);
3323  SDLoc dl = getCurSDLoc();
3324 
3325  // Normalize Vector GEP - all scalar operands should be converted to the
3326  // splat vector.
3327  unsigned VectorWidth = I.getType()->isVectorTy() ?
3328  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3329 
3330  if (VectorWidth && !N.getValueType().isVector()) {
3331  LLVMContext &Context = *DAG.getContext();
3332  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3333  N = DAG.getSplatBuildVector(VT, dl, N);
3334  }
3335 
3336  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3337  GTI != E; ++GTI) {
3338  const Value *Idx = GTI.getOperand();
3339  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3340  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3341  if (Field) {
3342  // N = N + Offset
3343  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3344 
3345  // In an inbounds GEP with an offset that is nonnegative even when
3346  // interpreted as signed, assume there is no unsigned overflow.
3348  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3349  Flags.setNoUnsignedWrap(true);
3350 
3351  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3352  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3353  }
3354  } else {
3355  MVT PtrTy =
3357  unsigned PtrSize = PtrTy.getSizeInBits();
3358  APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3359 
3360  // If this is a scalar constant or a splat vector of constants,
3361  // handle it quickly.
3362  const auto *CI = dyn_cast<ConstantInt>(Idx);
3363  if (!CI && isa<ConstantDataVector>(Idx) &&
3364  cast<ConstantDataVector>(Idx)->getSplatValue())
3365  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3366 
3367  if (CI) {
3368  if (CI->isZero())
3369  continue;
3370  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3371  LLVMContext &Context = *DAG.getContext();
3372  SDValue OffsVal = VectorWidth ?
3373  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3374  DAG.getConstant(Offs, dl, PtrTy);
3375 
3376  // In an inbouds GEP with an offset that is nonnegative even when
3377  // interpreted as signed, assume there is no unsigned overflow.
3379  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3380  Flags.setNoUnsignedWrap(true);
3381 
3382  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3383  continue;
3384  }
3385 
3386  // N = N + Idx * ElementSize;
3387  SDValue IdxN = getValue(Idx);
3388 
3389  if (!IdxN.getValueType().isVector() && VectorWidth) {
3390  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3391  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3392  }
3393 
3394  // If the index is smaller or larger than intptr_t, truncate or extend
3395  // it.
3396  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3397 
3398  // If this is a multiply by a power of two, turn it into a shl
3399  // immediately. This is a very common case.
3400  if (ElementSize != 1) {
3401  if (ElementSize.isPowerOf2()) {
3402  unsigned Amt = ElementSize.logBase2();
3403  IdxN = DAG.getNode(ISD::SHL, dl,
3404  N.getValueType(), IdxN,
3405  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3406  } else {
3407  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3408  IdxN = DAG.getNode(ISD::MUL, dl,
3409  N.getValueType(), IdxN, Scale);
3410  }
3411  }
3412 
3413  N = DAG.getNode(ISD::ADD, dl,
3414  N.getValueType(), N, IdxN);
3415  }
3416  }
3417 
3418  setValue(&I, N);
3419 }
3420 
3421 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3422  // If this is a fixed sized alloca in the entry block of the function,
3423  // allocate it statically on the stack.
3424  if (FuncInfo.StaticAllocaMap.count(&I))
3425  return; // getValue will auto-populate this.
3426 
3427  SDLoc dl = getCurSDLoc();
3428  Type *Ty = I.getAllocatedType();
3429  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3430  auto &DL = DAG.getDataLayout();
3431  uint64_t TySize = DL.getTypeAllocSize(Ty);
3432  unsigned Align =
3433  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3434 
3435  SDValue AllocSize = getValue(I.getArraySize());
3436 
3437  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3438  if (AllocSize.getValueType() != IntPtr)
3439  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3440 
3441  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3442  AllocSize,
3443  DAG.getConstant(TySize, dl, IntPtr));
3444 
3445  // Handle alignment. If the requested alignment is less than or equal to
3446  // the stack alignment, ignore it. If the size is greater than or equal to
3447  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3448  unsigned StackAlign =
3450  if (Align <= StackAlign)
3451  Align = 0;
3452 
3453  // Round the size of the allocation up to the stack alignment size
3454  // by add SA-1 to the size. This doesn't overflow because we're computing
3455  // an address inside an alloca.
3457  Flags.setNoUnsignedWrap(true);
3458  AllocSize = DAG.getNode(ISD::ADD, dl,
3459  AllocSize.getValueType(), AllocSize,
3460  DAG.getIntPtrConstant(StackAlign - 1, dl), Flags);
3461 
3462  // Mask out the low bits for alignment purposes.
3463  AllocSize = DAG.getNode(ISD::AND, dl,
3464  AllocSize.getValueType(), AllocSize,
3465  DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3466  dl));
3467 
3468  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3469  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3470  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3471  setValue(&I, DSA);
3472  DAG.setRoot(DSA.getValue(1));
3473 
3474  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3475 }
3476 
3477 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3478  if (I.isAtomic())
3479  return visitAtomicLoad(I);
3480 
3481  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3482  const Value *SV = I.getOperand(0);
3483  if (TLI.supportSwiftError()) {
3484  // Swifterror values can come from either a function parameter with
3485  // swifterror attribute or an alloca with swifterror attribute.
3486  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3487  if (Arg->hasSwiftErrorAttr())
3488  return visitLoadFromSwiftError(I);
3489  }
3490 
3491  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3492  if (Alloca->isSwiftError())
3493  return visitLoadFromSwiftError(I);
3494  }
3495  }
3496 
3497  SDValue Ptr = getValue(SV);
3498 
3499  Type *Ty = I.getType();
3500 
3501  bool isVolatile = I.isVolatile();
3502  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3503  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3504  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3505  unsigned Alignment = I.getAlignment();
3506 
3507  AAMDNodes AAInfo;
3508  I.getAAMetadata(AAInfo);
3509  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3510 
3513  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3514  unsigned NumValues = ValueVTs.size();
3515  if (NumValues == 0)
3516  return;
3517 
3518  SDValue Root;
3519  bool ConstantMemory = false;
3520  if (isVolatile || NumValues > MaxParallelChains)
3521  // Serialize volatile loads with other side effects.
3522  Root = getRoot();
3523  else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3524  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3525  // Do not serialize (non-volatile) loads of constant memory with anything.
3526  Root = DAG.getEntryNode();
3527  ConstantMemory = true;
3528  } else {
3529  // Do not serialize non-volatile loads against each other.
3530  Root = DAG.getRoot();
3531  }
3532 
3533  SDLoc dl = getCurSDLoc();
3534 
3535  if (isVolatile)
3536  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3537 
3538  // An aggregate load cannot wrap around the address space, so offsets to its
3539  // parts don't wrap either.
3541  Flags.setNoUnsignedWrap(true);
3542 
3543  SmallVector<SDValue, 4> Values(NumValues);
3544  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3545  EVT PtrVT = Ptr.getValueType();
3546  unsigned ChainI = 0;
3547  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3548  // Serializing loads here may result in excessive register pressure, and
3549  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3550  // could recover a bit by hoisting nodes upward in the chain by recognizing
3551  // they are side-effect free or do not alias. The optimizer should really
3552  // avoid this case by converting large object/array copies to llvm.memcpy
3553  // (MaxParallelChains should always remain as failsafe).
3554  if (ChainI == MaxParallelChains) {
3555  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3556  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3557  makeArrayRef(Chains.data(), ChainI));
3558  Root = Chain;
3559  ChainI = 0;
3560  }
3561  SDValue A = DAG.getNode(ISD::ADD, dl,
3562  PtrVT, Ptr,
3563  DAG.getConstant(Offsets[i], dl, PtrVT),
3564  Flags);
3565  auto MMOFlags = MachineMemOperand::MONone;
3566  if (isVolatile)
3567  MMOFlags |= MachineMemOperand::MOVolatile;
3568  if (isNonTemporal)
3570  if (isInvariant)
3571  MMOFlags |= MachineMemOperand::MOInvariant;
3572  if (isDereferenceable)
3574  MMOFlags |= TLI.getMMOFlags(I);
3575 
3576  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3577  MachinePointerInfo(SV, Offsets[i]), Alignment,
3578  MMOFlags, AAInfo, Ranges);
3579 
3580  Values[i] = L;
3581  Chains[ChainI] = L.getValue(1);
3582  }
3583 
3584  if (!ConstantMemory) {
3585  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3586  makeArrayRef(Chains.data(), ChainI));
3587  if (isVolatile)
3588  DAG.setRoot(Chain);
3589  else
3590  PendingLoads.push_back(Chain);
3591  }
3592 
3593  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3594  DAG.getVTList(ValueVTs), Values));
3595 }
3596 
3597 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3599  "call visitStoreToSwiftError when backend supports swifterror");
3600 
3603  const Value *SrcV = I.getOperand(0);
3605  SrcV->getType(), ValueVTs, &Offsets);
3606  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3607  "expect a single EVT for swifterror");
3608 
3609  SDValue Src = getValue(SrcV);
3610  // Create a virtual register, then update the virtual register.
3611  unsigned VReg; bool CreatedVReg;
3612  std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3613  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3614  // Chain can be getRoot or getControlRoot.
3615  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3616  SDValue(Src.getNode(), Src.getResNo()));
3617  DAG.setRoot(CopyNode);
3618  if (CreatedVReg)
3619  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3620 }
3621 
3622 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3624  "call visitLoadFromSwiftError when backend supports swifterror");
3625 
3626  assert(!I.isVolatile() &&
3627  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3629  "Support volatile, non temporal, invariant for load_from_swift_error");
3630 
3631  const Value *SV = I.getOperand(0);
3632  Type *Ty = I.getType();
3633  AAMDNodes AAInfo;
3634  I.getAAMetadata(AAInfo);
3635  assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
3636  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
3637  "load_from_swift_error should not be constant memory");
3638 
3642  ValueVTs, &Offsets);
3643  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3644  "expect a single EVT for swifterror");
3645 
3646  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3647  SDValue L = DAG.getCopyFromReg(
3648  getRoot(), getCurSDLoc(),
3649  FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3650  ValueVTs[0]);
3651 
3652  setValue(&I, L);
3653 }
3654 
3655 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3656  if (I.isAtomic())
3657  return visitAtomicStore(I);
3658 
3659  const Value *SrcV = I.getOperand(0);
3660  const Value *PtrV = I.getOperand(1);
3661 
3662  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3663  if (TLI.supportSwiftError()) {
3664  // Swifterror values can come from either a function parameter with
3665  // swifterror attribute or an alloca with swifterror attribute.
3666  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3667  if (Arg->hasSwiftErrorAttr())
3668  return visitStoreToSwiftError(I);
3669  }
3670 
3671  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3672  if (Alloca->isSwiftError())
3673  return visitStoreToSwiftError(I);
3674  }
3675  }
3676 
3680  SrcV->getType(), ValueVTs, &Offsets);
3681  unsigned NumValues = ValueVTs.size();
3682  if (NumValues == 0)
3683  return;
3684 
3685  // Get the lowered operands. Note that we do this after
3686  // checking if NumResults is zero, because with zero results
3687  // the operands won't have values in the map.
3688  SDValue Src = getValue(SrcV);
3689  SDValue Ptr = getValue(PtrV);
3690 
3691  SDValue Root = getRoot();
3692  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3693  SDLoc dl = getCurSDLoc();
3694  EVT PtrVT = Ptr.getValueType();
3695  unsigned Alignment = I.getAlignment();
3696  AAMDNodes AAInfo;
3697  I.getAAMetadata(AAInfo);
3698 
3699  auto MMOFlags = MachineMemOperand::MONone;
3700  if (I.isVolatile())
3701  MMOFlags |= MachineMemOperand::MOVolatile;
3702  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3704  MMOFlags |= TLI.getMMOFlags(I);
3705 
3706  // An aggregate load cannot wrap around the address space, so offsets to its
3707  // parts don't wrap either.
3709  Flags.setNoUnsignedWrap(true);
3710 
3711  unsigned ChainI = 0;
3712  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3713  // See visitLoad comments.
3714  if (ChainI == MaxParallelChains) {
3715  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3716  makeArrayRef(Chains.data(), ChainI));
3717  Root = Chain;
3718  ChainI = 0;
3719  }
3720  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3721  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3722  SDValue St = DAG.getStore(
3723  Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3724  MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3725  Chains[ChainI] = St;
3726  }
3727 
3728  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3729  makeArrayRef(Chains.data(), ChainI));
3730  DAG.setRoot(StoreNode);
3731 }
3732 
3733 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3734  bool IsCompressing) {
3735  SDLoc sdl = getCurSDLoc();
3736 
3737  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3738  unsigned& Alignment) {
3739  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3740  Src0 = I.getArgOperand(0);
3741  Ptr = I.getArgOperand(1);
3742  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3743  Mask = I.getArgOperand(3);
3744  };
3745  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3746  unsigned& Alignment) {
3747  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3748  Src0 = I.getArgOperand(0);
3749  Ptr = I.getArgOperand(1);
3750  Mask = I.getArgOperand(2);
3751  Alignment = 0;
3752  };
3753 
3754  Value *PtrOperand, *MaskOperand, *Src0Operand;
3755  unsigned Alignment;
3756  if (IsCompressing)
3757  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3758  else
3759  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3760 
3761  SDValue Ptr = getValue(PtrOperand);
3762  SDValue Src0 = getValue(Src0Operand);
3763  SDValue Mask = getValue(MaskOperand);
3764 
3765  EVT VT = Src0.getValueType();
3766  if (!Alignment)
3767  Alignment = DAG.getEVTAlignment(VT);
3768 
3769  AAMDNodes AAInfo;
3770  I.getAAMetadata(AAInfo);
3771 
3772  MachineMemOperand *MMO =
3773  DAG.getMachineFunction().
3774  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3776  Alignment, AAInfo);
3777  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3778  MMO, false /* Truncating */,
3779  IsCompressing);
3780  DAG.setRoot(StoreNode);
3781  setValue(&I, StoreNode);
3782 }
3783 
3784 // Get a uniform base for the Gather/Scatter intrinsic.
3785 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3786 // We try to represent it as a base pointer + vector of indices.
3787 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3788 // The first operand of the GEP may be a single pointer or a vector of pointers
3789 // Example:
3790 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3791 // or
3792 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3793 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3794 //
3795 // When the first GEP operand is a single pointer - it is the uniform base we
3796 // are looking for. If first operand of the GEP is a splat vector - we
3797 // extract the spalt value and use it as a uniform base.
3798 // In all other cases the function returns 'false'.
3799 //
3800 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3801  SelectionDAGBuilder* SDB) {
3802 
3803  SelectionDAG& DAG = SDB->DAG;
3804  LLVMContext &Context = *DAG.getContext();
3805 
3806  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3808  if (!GEP || GEP->getNumOperands() > 2)
3809  return false;
3810 
3811  const Value *GEPPtr = GEP->getPointerOperand();
3812  if (!GEPPtr->getType()->isVectorTy())
3813  Ptr = GEPPtr;
3814  else if (!(Ptr = getSplatValue(GEPPtr)))
3815  return false;
3816 
3817  Value *IndexVal = GEP->getOperand(1);
3818 
3819  // The operands of the GEP may be defined in another basic block.
3820  // In this case we'll not find nodes for the operands.
3821  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3822  return false;
3823 
3824  Base = SDB->getValue(Ptr);
3825  Index = SDB->getValue(IndexVal);
3826 
3827  // Suppress sign extension.
3828  if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3829  if (SDB->findValue(Sext->getOperand(0))) {
3830  IndexVal = Sext->getOperand(0);
3831  Index = SDB->getValue(IndexVal);
3832  }
3833  }
3834  if (!Index.getValueType().isVector()) {
3835  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3836  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3837  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3838  }
3839  return true;
3840 }
3841 
3842 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3843  SDLoc sdl = getCurSDLoc();
3844 
3845  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3846  const Value *Ptr = I.getArgOperand(1);
3847  SDValue Src0 = getValue(I.getArgOperand(0));
3848  SDValue Mask = getValue(I.getArgOperand(3));
3849  EVT VT = Src0.getValueType();
3850  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3851  if (!Alignment)
3852  Alignment = DAG.getEVTAlignment(VT);
3853  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3854 
3855  AAMDNodes AAInfo;
3856  I.getAAMetadata(AAInfo);
3857 
3858  SDValue Base;
3859  SDValue Index;
3860  const Value *BasePtr = Ptr;
3861  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3862 
3863  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3865  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3866  MachineMemOperand::MOStore, VT.getStoreSize(),
3867  Alignment, AAInfo);
3868  if (!UniformBase) {
3869  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3870  Index = getValue(Ptr);
3871  }
3872  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3873  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3874  Ops, MMO);
3875  DAG.setRoot(Scatter);
3876  setValue(&I, Scatter);
3877 }
3878 
3879 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3880  SDLoc sdl = getCurSDLoc();
3881 
3882  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3883  unsigned& Alignment) {
3884  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3885  Ptr = I.getArgOperand(0);
3886  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3887  Mask = I.getArgOperand(2);
3888  Src0 = I.getArgOperand(3);
3889  };
3890  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3891  unsigned& Alignment) {
3892  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3893  Ptr = I.getArgOperand(0);
3894  Alignment = 0;
3895  Mask = I.getArgOperand(1);
3896  Src0 = I.getArgOperand(2);
3897  };
3898 
3899  Value *PtrOperand, *MaskOperand, *Src0Operand;
3900  unsigned Alignment;
3901  if (IsExpanding)
3902  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3903  else
3904  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3905 
3906  SDValue Ptr = getValue(PtrOperand);
3907  SDValue Src0 = getValue(Src0Operand);
3908  SDValue Mask = getValue(MaskOperand);
3909 
3910  EVT VT = Src0.getValueType();
3911  if (!Alignment)
3912  Alignment = DAG.getEVTAlignment(VT);
3913 
3914  AAMDNodes AAInfo;
3915  I.getAAMetadata(AAInfo);
3916  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3917 
3918  // Do not serialize masked loads of constant memory with anything.
3919  bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
3920  PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3921  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3922 
3923  MachineMemOperand *MMO =
3924  DAG.getMachineFunction().
3925  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3927  Alignment, AAInfo, Ranges);
3928 
3929  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3930  ISD::NON_EXTLOAD, IsExpanding);
3931  if (AddToChain) {
3932  SDValue OutChain = Load.getValue(1);
3933  DAG.setRoot(OutChain);
3934  }
3935  setValue(&I, Load);
3936 }
3937 
3938 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3939  SDLoc sdl = getCurSDLoc();
3940 
3941  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3942  const Value *Ptr = I.getArgOperand(0);
3943  SDValue Src0 = getValue(I.getArgOperand(3));
3944  SDValue Mask = getValue(I.getArgOperand(2));
3945 
3946  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3947  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3948  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3949  if (!Alignment)
3950  Alignment = DAG.getEVTAlignment(VT);
3951 
3952  AAMDNodes AAInfo;
3953  I.getAAMetadata(AAInfo);
3954  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3955 
3956  SDValue Root = DAG.getRoot();
3957  SDValue Base;
3958  SDValue Index;
3959  const Value *BasePtr = Ptr;
3960  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3961  bool ConstantMemory = false;
3962  if (UniformBase &&
3963  AA && AA->pointsToConstantMemory(MemoryLocation(
3964  BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3965  AAInfo))) {
3966  // Do not serialize (non-volatile) loads of constant memory with anything.
3967  Root = DAG.getEntryNode();
3968  ConstantMemory = true;
3969  }
3970 
3971  MachineMemOperand *MMO =
3972  DAG.getMachineFunction().
3973  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3975  Alignment, AAInfo, Ranges);
3976 
3977  if (!UniformBase) {
3978  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3979  Index = getValue(Ptr);
3980  }
3981  SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3982  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3983  Ops, MMO);
3984 
3985  SDValue OutChain = Gather.getValue(1);
3986  if (!ConstantMemory)
3987  PendingLoads.push_back(OutChain);
3988  setValue(&I, Gather);
3989 }
3990 
3991 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3992  SDLoc dl = getCurSDLoc();
3993  AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3994  AtomicOrdering FailureOrder = I.getFailureOrdering();
3995  SyncScope::ID SSID = I.getSyncScopeID();
3996 
3997  SDValue InChain = getRoot();
3998 
3999  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4000  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4001  SDValue L = DAG.getAtomicCmpSwap(
4002  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4003  getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4005  /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4006 
4007  SDValue OutChain = L.getValue(2);
4008 
4009  setValue(&I, L);
4010  DAG.setRoot(OutChain);
4011 }
4012 
4013 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4014  SDLoc dl = getCurSDLoc();
4015  ISD::NodeType NT;
4016  switch (I.getOperation()) {
4017  default: llvm_unreachable("Unknown atomicrmw operation");
4018  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4019  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4020  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4021  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4022  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4023  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4024  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4025  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4026  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4027  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4028  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4029  }
4030  AtomicOrdering Order = I.getOrdering();
4031  SyncScope::ID SSID = I.getSyncScopeID();
4032 
4033  SDValue InChain = getRoot();
4034 
4035  SDValue L =
4036  DAG.getAtomic(NT, dl,
4037  getValue(I.getValOperand()).getSimpleValueType(),
4038  InChain,
4039  getValue(I.getPointerOperand()),
4040  getValue(I.getValOperand()),
4041  I.getPointerOperand(),
4042  /* Alignment=*/ 0, Order, SSID);
4043 
4044  SDValue OutChain = L.getValue(1);
4045 
4046  setValue(&I, L);
4047  DAG.setRoot(OutChain);
4048 }
4049 
4050 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4051  SDLoc dl = getCurSDLoc();
4052  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4053  SDValue Ops[3];
4054  Ops[0] = getRoot();
4055  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4056  TLI.getFenceOperandTy(DAG.getDataLayout()));
4057  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4058  TLI.getFenceOperandTy(DAG.getDataLayout()));
4059  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4060 }
4061 
4062 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4063  SDLoc dl = getCurSDLoc();
4064  AtomicOrdering Order = I.getOrdering();
4065  SyncScope::ID SSID = I.getSyncScopeID();
4066 
4067  SDValue InChain = getRoot();
4068 
4069  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4070  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4071 
4072  if (I.getAlignment() < VT.getSizeInBits() / 8)
4073  report_fatal_error("Cannot generate unaligned atomic load");
4074 
4075  MachineMemOperand *MMO =
4076  DAG.getMachineFunction().
4077  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4080  VT.getStoreSize(),
4081  I.getAlignment() ? I.getAlignment() :
4082  DAG.getEVTAlignment(VT),
4083  AAMDNodes(), nullptr, SSID, Order);
4084 
4085  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4086  SDValue L =
4087  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4088  getValue(I.getPointerOperand()), MMO);
4089 
4090  SDValue OutChain = L.getValue(1);
4091 
4092  setValue(&I, L);
4093  DAG.setRoot(OutChain);
4094 }
4095 
4096 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4097  SDLoc dl = getCurSDLoc();
4098 
4099  AtomicOrdering Order = I.getOrdering();
4100  SyncScope::ID SSID = I.getSyncScopeID();
4101 
4102  SDValue InChain = getRoot();
4103 
4104  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4105  EVT VT =
4107 
4108  if (I.getAlignment() < VT.getSizeInBits() / 8)
4109  report_fatal_error("Cannot generate unaligned atomic store");
4110 
4111  SDValue OutChain =
4112  DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4113  InChain,
4114  getValue(I.getPointerOperand()),
4115  getValue(I.getValueOperand()),
4117  Order, SSID);
4118 
4119  DAG.setRoot(OutChain);
4120 }
4121 
4122 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4123 /// node.
4124 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4125  unsigned Intrinsic) {
4126  // Ignore the callsite's attributes. A specific call site may be marked with
4127  // readnone, but the lowering code will expect the chain based on the
4128  // definition.
4129  const Function *F = I.getCalledFunction();
4130  bool HasChain = !F->doesNotAccessMemory();
4131  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4132 
4133  // Build the operand list.
4135  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4136  if (OnlyLoad) {
4137  // We don't need to serialize loads against other loads.
4138  Ops.push_back(DAG.getRoot());
4139  } else {
4140  Ops.push_back(getRoot());
4141  }
4142  }
4143 
4144  // Info is set by getTgtMemInstrinsic
4145  TargetLowering::IntrinsicInfo Info;
4146  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4147  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4148 
4149  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4150  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4151  Info.opc == ISD::INTRINSIC_W_CHAIN)
4152  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4153  TLI.getPointerTy(DAG.getDataLayout())));
4154 
4155  // Add all operands of the call to the operand list.
4156  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4157  SDValue Op = getValue(I.getArgOperand(i));
4158  Ops.push_back(Op);
4159  }
4160 
4162  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4163 
4164  if (HasChain)
4165  ValueVTs.push_back(MVT::Other);
4166 
4167  SDVTList VTs = DAG.getVTList(ValueVTs);
4168 
4169  // Create the node.
4170  SDValue Result;
4171  if (IsTgtIntrinsic) {
4172  // This is target intrinsic that touches memory
4173  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4174  VTs, Ops, Info.memVT,
4175  MachinePointerInfo(Info.ptrVal, Info.offset),
4176  Info.align, Info.vol,
4177  Info.readMem, Info.writeMem, Info.size);
4178  } else if (!HasChain) {
4179  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4180  } else if (!I.getType()->isVoidTy()) {
4181  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4182  } else {
4183  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4184  }
4185 
4186  if (HasChain) {
4187  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4188  if (OnlyLoad)
4189  PendingLoads.push_back(Chain);
4190  else
4191  DAG.setRoot(Chain);
4192  }
4193 
4194  if (!I.getType()->isVoidTy()) {
4195  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4196  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4197  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4198  } else
4199  Result = lowerRangeToAssertZExt(DAG, I, Result);
4200 
4201  setValue(&I, Result);
4202  }
4203 }
4204 
4205 /// GetSignificand - Get the significand and build it into a floating-point
4206 /// number with exponent of 1:
4207 ///
4208 /// Op = (Op & 0x007fffff) | 0x3f800000;
4209 ///
4210 /// where Op is the hexadecimal representation of floating point value.
4212  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4213  DAG.getConstant(0x007fffff, dl, MVT::i32));
4214  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4215  DAG.getConstant(0x3f800000, dl, MVT::i32));
4216  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4217 }
4218 
4219 /// GetExponent - Get the exponent:
4220 ///
4221 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4222 ///
4223 /// where Op is the hexadecimal representation of floating point value.
4225  const TargetLowering &TLI, const SDLoc &dl) {
4226  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4227  DAG.getConstant(0x7f800000, dl, MVT::i32));
4228  SDValue t1 = DAG.getNode(
4229  ISD::SRL, dl, MVT::i32, t0,
4230  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4231  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4232  DAG.getConstant(127, dl, MVT::i32));
4233  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4234 }
4235 
4236 /// getF32Constant - Get 32-bit floating point constant.
4237 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4238  const SDLoc &dl) {
4239  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4240  MVT::f32);
4241 }
4242 
4244  SelectionDAG &DAG) {
4245  // TODO: What fast-math-flags should be set on the floating-point nodes?
4246 
4247  // IntegerPartOfX = ((int32_t)(t0);
4248  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4249 
4250  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4251  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4252  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4253 
4254  // IntegerPartOfX <<= 23;
4255  IntegerPartOfX = DAG.getNode(
4256  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4258  DAG.getDataLayout())));
4259 
4260  SDValue TwoToFractionalPartOfX;
4261  if (LimitFloatPrecision <= 6) {
4262  // For floating-point precision of 6:
4263  //
4264  // TwoToFractionalPartOfX =
4265  // 0.997535578f +
4266  // (0.735607626f + 0.252464424f * x) * x;
4267  //
4268  // error 0.0144103317, which is 6 bits
4269  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4270  getF32Constant(DAG, 0x3e814304, dl));
4271  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4272  getF32Constant(DAG, 0x3f3c50c8, dl));
4273  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4274  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4275  getF32Constant(DAG, 0x3f7f5e7e, dl));
4276  } else if (LimitFloatPrecision <= 12) {
4277  // For floating-point precision of 12:
4278  //
4279  // TwoToFractionalPartOfX =
4280  // 0.999892986f +
4281  // (0.696457318f +
4282  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4283  //
4284  // error 0.000107046256, which is 13 to 14 bits
4285  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4286  getF32Constant(DAG, 0x3da235e3, dl));
4287  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4288  getF32Constant(DAG, 0x3e65b8f3, dl));
4289  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4290  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4291  getF32Constant(DAG, 0x3f324b07, dl));
4292  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4293  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4294  getF32Constant(DAG, 0x3f7ff8fd, dl));
4295  } else { // LimitFloatPrecision <= 18
4296  // For floating-point precision of 18:
4297  //
4298  // TwoToFractionalPartOfX =
4299  // 0.999999982f +
4300  // (0.693148872f +
4301  // (0.240227044f +
4302  // (0.554906021e-1f +
4303  // (0.961591928e-2f +
4304  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4305  // error 2.47208000*10^(-7), which is better than 18 bits
4306  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4307  getF32Constant(DAG, 0x3924b03e, dl));
4308  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4309  getF32Constant(DAG, 0x3ab24b87, dl));
4310  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4311  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4312  getF32Constant(DAG, 0x3c1d8c17, dl));
4313  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4314  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4315  getF32Constant(DAG, 0x3d634a1d, dl));
4316  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4317  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4318  getF32Constant(DAG, 0x3e75fe14, dl));
4319  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4320  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4321  getF32Constant(DAG, 0x3f317234, dl));
4322  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4323  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4324  getF32Constant(DAG, 0x3f800000, dl));
4325  }
4326 
4327  // Add the exponent into the result in integer domain.
4328  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4329  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4330  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4331 }
4332 
4333 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4334 /// limited-precision mode.
4335 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4336  const TargetLowering &TLI) {
4337  if (Op.getValueType() == MVT::f32 &&
4338  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4339 
4340  // Put the exponent in the right bit position for later addition to the
4341  // final result:
4342  //
4343  // #define LOG2OFe 1.4426950f
4344  // t0 = Op * LOG2OFe
4345 
4346  // TODO: What fast-math-flags should be set here?
4347  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4348  getF32Constant(DAG, 0x3fb8aa3b, dl));
4349  return getLimitedPrecisionExp2(t0, dl, DAG);
4350  }
4351 
4352  // No special expansion.
4353  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4354 }
4355 
4356 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4357 /// limited-precision mode.
4358 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4359  const TargetLowering &TLI) {
4360 
4361  // TODO: What fast-math-flags should be set on the floating-point nodes?
4362 
4363  if (Op.getValueType() == MVT::f32 &&
4364  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4365  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4366 
4367  // Scale the exponent by log(2) [0.69314718f].
4368  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4369  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4370  getF32Constant(DAG, 0x3f317218, dl));
4371 
4372  // Get the significand and build it into a floating-point number with
4373  // exponent of 1.
4374  SDValue X = GetSignificand(DAG, Op1, dl);
4375 
4376  SDValue LogOfMantissa;
4377  if (LimitFloatPrecision <= 6) {
4378  // For floating-point precision of 6:
4379  //
4380  // LogofMantissa =
4381  // -1.1609546f +
4382  // (1.4034025f - 0.23903021f * x) * x;
4383  //
4384  // error 0.0034276066, which is better than 8 bits
4385  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4386  getF32Constant(DAG, 0xbe74c456, dl));
4387  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4388  getF32Constant(DAG, 0x3fb3a2b1, dl));
4389  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4390  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4391  getF32Constant(DAG, 0x3f949a29, dl));
4392  } else if (LimitFloatPrecision <= 12) {
4393  // For floating-point precision of 12:
4394  //
4395  // LogOfMantissa =
4396  // -1.7417939f +
4397  // (2.8212026f +
4398  // (-1.4699568f +
4399  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4400  //
4401  // error 0.000061011436, which is 14 bits
4402  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4403  getF32Constant(DAG, 0xbd67b6d6, dl));
4404  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4405  getF32Constant(DAG, 0x3ee4f4b8, dl));
4406  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4407  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4408  getF32Constant(DAG, 0x3fbc278b, dl));
4409  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4410  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4411  getF32Constant(DAG, 0x40348e95, dl));
4412  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4413  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4414  getF32Constant(DAG, 0x3fdef31a, dl));
4415  } else { // LimitFloatPrecision <= 18
4416  // For floating-point precision of 18:
4417  //
4418  // LogOfMantissa =
4419  // -2.1072184f +
4420  // (4.2372794f +
4421  // (-3.7029485f +
4422  // (2.2781945f +
4423  // (-0.87823314f +
4424  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4425  //
4426  // error 0.0000023660568, which is better than 18 bits
4427  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4428  getF32Constant(DAG, 0xbc91e5ac, dl));
4429  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4430  getF32Constant(DAG, 0x3e4350aa, dl));
4431  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4432  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4433  getF32Constant(DAG, 0x3f60d3e3, dl));
4434  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4435  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4436  getF32Constant(DAG, 0x4011cdf0, dl));
4437  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4438  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4439  getF32Constant(DAG, 0x406cfd1c, dl));
4440  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4441  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4442  getF32Constant(DAG, 0x408797cb, dl));
4443  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4444  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4445  getF32Constant(DAG, 0x4006dcab, dl));
4446  }
4447 
4448  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4449  }
4450 
4451  // No special expansion.
4452  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4453 }
4454 
4455 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4456 /// limited-precision mode.
4457 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4458  const TargetLowering &TLI) {
4459 
4460  // TODO: What fast-math-flags should be set on the floating-point nodes?
4461 
4462  if (Op.getValueType() == MVT::f32 &&
4463  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4464  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4465 
4466  // Get the exponent.
4467  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4468 
4469  // Get the significand and build it into a floating-point number with
4470  // exponent of 1.
4471  SDValue X = GetSignificand(DAG, Op1, dl);
4472 
4473  // Different possible minimax approximations of significand in
4474  // floating-point for various degrees of accuracy over [1,2].
4475  SDValue Log2ofMantissa;
4476  if (LimitFloatPrecision <= 6) {
4477  // For floating-point precision of 6:
4478  //
4479  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4480  //
4481  // error 0.0049451742, which is more than 7 bits
4482  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4483  getF32Constant(DAG, 0xbeb08fe0, dl));
4484  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4485  getF32Constant(DAG, 0x40019463, dl));
4486  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4487  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4488  getF32Constant(DAG, 0x3fd6633d, dl));
4489  } else if (LimitFloatPrecision <= 12) {
4490  // For floating-point precision of 12:
4491  //
4492  // Log2ofMantissa =
4493  // -2.51285454f +
4494  // (4.07009056f +
4495  // (-2.12067489f +
4496  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4497  //
4498  // error 0.0000876136000, which is better than 13 bits
4499  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4500  getF32Constant(DAG, 0xbda7262e, dl));
4501  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4502  getF32Constant(DAG, 0x3f25280b, dl));
4503  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4504  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4505  getF32Constant(DAG, 0x4007b923, dl));
4506  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4507  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4508  getF32Constant(DAG, 0x40823e2f, dl));
4509  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4510  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4511  getF32Constant(DAG, 0x4020d29c, dl));
4512  } else { // LimitFloatPrecision <= 18
4513  // For floating-point precision of 18:
4514  //
4515  // Log2ofMantissa =
4516  // -3.0400495f +
4517  // (6.1129976f +
4518  // (-5.3420409f +
4519  // (3.2865683f +
4520  // (-1.2669343f +
4521  // (0.27515199f -
4522  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4523  //
4524  // error 0.0000018516, which is better than 18 bits
4525  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4526  getF32Constant(DAG, 0xbcd2769e, dl));
4527  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4528  getF32Constant(DAG, 0x3e8ce0b9, dl));
4529  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4530  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4531  getF32Constant(DAG, 0x3fa22ae7, dl));
4532  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4533  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4534  getF32Constant(DAG, 0x40525723, dl));
4535  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4536  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4537  getF32Constant(DAG, 0x40aaf200, dl));
4538  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4539  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4540  getF32Constant(DAG, 0x40c39dad, dl));
4541  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4542  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4543  getF32Constant(DAG, 0x4042902c, dl));
4544  }
4545 
4546  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4547  }
4548 
4549  // No special expansion.
4550  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4551 }
4552 
4553 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4554 /// limited-precision mode.
4556  const TargetLowering &TLI) {
4557 
4558  // TODO: What fast-math-flags should be set on the floating-point nodes?
4559 
4560  if (Op.getValueType() == MVT::f32 &&
4561  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4562  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4563 
4564  // Scale the exponent by log10(2) [0.30102999f].
4565  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4566  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4567  getF32Constant(DAG, 0x3e9a209a, dl));
4568 
4569  // Get the significand and build it into a floating-point number with
4570  // exponent of 1.
4571  SDValue X = GetSignificand(DAG, Op1, dl);
4572 
4573  SDValue Log10ofMantissa;
4574  if (LimitFloatPrecision <= 6) {
4575  // For floating-point precision of 6:
4576  //
4577  // Log10ofMantissa =
4578  // -0.50419619f +
4579  // (0.60948995f - 0.10380950f * x) * x;
4580  //
4581  // error 0.0014886165, which is 6 bits
4582  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4583  getF32Constant(DAG, 0xbdd49a13, dl));
4584  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4585  getF32Constant(DAG, 0x3f1c0789, dl));
4586  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4587  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4588  getF32Constant(DAG, 0x3f011300, dl));
4589  } else if (LimitFloatPrecision <= 12) {
4590  // For floating-point precision of 12:
4591  //
4592  // Log10ofMantissa =
4593  // -0.64831180f +
4594  // (0.91751397f +
4595  // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4596  //
4597  // error 0.00019228036, which is better than 12 bits
4598  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4599  getF32Constant(DAG, 0x3d431f31, dl));
4600  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4601  getF32Constant(DAG, 0x3ea21fb2, dl));
4602  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4603  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4604  getF32Constant(DAG, 0x3f6ae232, dl));
4605  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4606  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4607  getF32Constant(DAG, 0x3f25f7c3, dl));
4608  } else { // LimitFloatPrecision <= 18
4609  // For floating-point precision of 18:
4610  //
4611  // Log10ofMantissa =
4612  // -0.84299375f +
4613  // (1.5327582f +
4614  // (-1.0688956f +
4615  // (0.49102474f +
4616  // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4617  //
4618  // error 0.0000037995730, which is better than 18 bits
4619  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4620  getF32Constant(DAG, 0x3c5d51ce, dl));
4621  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4622  getF32Constant(DAG, 0x3e00685a, dl));
4623  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4624  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4625  getF32Constant(DAG, 0x3efb6798, dl));
4626  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4627  SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4628  getF32Constant(DAG, 0x3f88d192, dl));
4629  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4630  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4631  getF32Constant(DAG, 0x3fc4316c, dl));
4632  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4633  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4634  getF32Constant(DAG, 0x3f57ce70, dl));
4635  }
4636 
4637  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4638  }
4639 
4640  // No special expansion.
4641  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4642 }
4643 
4644 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4645 /// limited-precision mode.
4646 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4647  const TargetLowering &TLI) {
4648  if (Op.getValueType() == MVT::f32 &&
4649  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4650  return getLimitedPrecisionExp2(Op, dl, DAG);
4651 
4652  // No special expansion.
4653  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4654 }
4655 
4656 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4657 /// limited-precision mode with x == 10.0f.
4658 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4659  SelectionDAG &DAG, const TargetLowering &TLI) {
4660  bool IsExp10 = false;
4661  if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4662  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4663  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4664  APFloat Ten(10.0f);
4665  IsExp10 = LHSC->isExactlyValue(Ten);
4666  }
4667  }
4668 
4669  // TODO: What fast-math-flags should be set on the FMUL node?
4670  if (IsExp10) {
4671  // Put the exponent in the right bit position for later addition to the
4672  // final result:
4673  //
4674  // #define LOG2OF10 3.3219281f
4675  // t0 = Op * LOG2OF10;
4676  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4677  getF32Constant(DAG, 0x40549a78, dl));
4678  return getLimitedPrecisionExp2(t0, dl, DAG);
4679  }
4680 
4681  // No special expansion.
4682  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4683 }
4684 
4685 
4686 /// ExpandPowI - Expand a llvm.powi intrinsic.
4687 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4688  SelectionDAG &DAG) {
4689  // If RHS is a constant, we can expand this out to a multiplication tree,
4690  // otherwise we end up lowering to a call to __powidf2 (for example). When
4691  // optimizing for size, we only want to do this if the expansion would produce
4692  // a small number of multiplies, otherwise we do the full expansion.
4693  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4694  // Get the exponent as a positive value.
4695  unsigned Val = RHSC->getSExtValue();
4696  if ((int)Val < 0) Val = -Val;
4697 
4698  // powi(x, 0) -> 1.0
4699  if (Val == 0)
4700  return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4701 
4702  const Function *F = DAG.getMachineFunction().getFunction();
4703  if (!F->optForSize() ||
4704  // If optimizing for size, don't insert too many multiplies.
4705  // This inserts up to 5 multiplies.
4706  countPopulation(Val) + Log2_32(Val) < 7) {
4707  // We use the simple binary decomposition method to generate the multiply
4708  // sequence. There are more optimal ways to do this (for example,
4709  // powi(x,15) generates one more multiply than it should), but this has
4710  // the benefit of being both really simple and much better than a libcall.
4711  SDValue Res; // Logically starts equal to 1.0
4712  SDValue CurSquare = LHS;
4713  // TODO: Intrinsics should have fast-math-flags that propagate to these
4714  // nodes.
4715  while (Val) {
4716  if (Val & 1) {
4717  if (Res.getNode())
4718  Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4719  else
4720  Res = CurSquare; // 1.0*CurSquare.
4721  }
4722 
4723  CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4724  CurSquare, CurSquare);
4725  Val >>= 1;
4726  }
4727 
4728  // If the original was negative, invert the result, producing 1/(x*x*x).
4729  if (RHSC->getSExtValue() < 0)
4730  Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4731  DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4732  return Res;
4733  }
4734  }
4735 
4736  // Otherwise, expand to a libcall.
4737  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4738 }
4739 
4740 // getUnderlyingArgReg - Find underlying register used for a truncated or
4741 // bitcasted argument.
4742 static unsigned getUnderlyingArgReg(const SDValue &N) {
4743  switch (N.getOpcode()) {
4744  case ISD::CopyFromReg:
4745  return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4746  case ISD::BITCAST:
4747  case ISD::AssertZext:
4748  case ISD::AssertSext:
4749  case ISD::TRUNCATE:
4750  return getUnderlyingArgReg(N.getOperand(0));
4751  default:
4752  return 0;
4753  }
4754 }
4755 
4756 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4757 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
4758 /// At the end of instruction selection, they will be inserted to the entry BB.
4759 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4760  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4761  DILocation *DL, int64_t Offset, bool IsDbgDeclare, const SDValue &N) {
4762  const Argument *Arg = dyn_cast<Argument>(V);
4763  if (!Arg)
4764  return false;
4765 
4766  MachineFunction &MF = DAG.getMachineFunction();
4767  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4768 
4769  // Ignore inlined function arguments here.
4770  //
4771  // FIXME: Should we be checking DL->inlinedAt() to determine this?
4772  if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4773  return false;
4774 
4775  bool IsIndirect = false;
4777  // Some arguments' frame index is recorded during argument lowering.
4778  int FI = FuncInfo.getArgumentFrameIndex(Arg);
4779  if (FI != INT_MAX)
4780  Op = MachineOperand::CreateFI(FI);
4781 
4782  if (!Op && N.getNode()) {
4783  unsigned Reg = getUnderlyingArgReg(N);
4784  if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4785  MachineRegisterInfo &RegInfo = MF.getRegInfo();
4786  unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4787  if (PR)
4788  Reg = PR;
4789  }
4790  if (Reg) {
4791  Op = MachineOperand::CreateReg(Reg, false);
4792  IsIndirect = IsDbgDeclare;
4793  }
4794  }
4795 
4796  if (!Op) {
4797  // Check if ValueMap has reg number.
4798  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4799  if (VMI != FuncInfo.ValueMap.end()) {
4800  Op = MachineOperand::CreateReg(VMI->second, false);
4801  IsIndirect = IsDbgDeclare;
4802  }
4803  }
4804 
4805  if (!Op && N.getNode())
4806  // Check if frame index is available.
4807  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4808  if (FrameIndexSDNode *FINode =
4809  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4810  Op = MachineOperand::CreateFI(FINode->getIndex());
4811 
4812  if (!Op)
4813  return false;
4814 
4815  assert(Variable->isValidLocationForIntrinsic(DL) &&
4816  "Expected inlined-at fields to agree");
4817  if (Op->isReg())
4818  FuncInfo.ArgDbgValues.push_back(
4819  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4820  Op->getReg(), Offset, Variable, Expr));
4821  else
4822  FuncInfo.ArgDbgValues.push_back(
4823  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4824  .add(*Op)
4825  .addImm(Offset)
4826  .addMetadata(Variable)
4827  .addMetadata(Expr));
4828 
4829  return true;
4830 }
4831 
4832 /// Return the appropriate SDDbgValue based on N.
4833 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4834  DILocalVariable *Variable,
4835  DIExpression *Expr, int64_t Offset,
4836  const DebugLoc &dl,
4837  unsigned DbgSDNodeOrder) {
4838  if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
4839  // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4840  // stack slot locations as such instead of as indirectly addressed
4841  // locations.
4842  return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), 0, dl,
4843  DbgSDNodeOrder);
4844  }
4845  return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4846  Offset, dl, DbgSDNodeOrder);
4847 }
4848 
4849 // VisualStudio defines setjmp as _setjmp
4850 #if defined(_MSC_VER) && defined(setjmp) && \
4851  !defined(setjmp_undefined_for_msvc)
4852 # pragma push_macro("setjmp")
4853 # undef setjmp
4854 # define setjmp_undefined_for_msvc
4855 #endif
4856 
4857 /// Lower the call to the specified intrinsic function. If we want to emit this
4858 /// as a call to a named external function, return the name. Otherwise, lower it
4859 /// and return null.
4860 const char *
4861 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4862  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4863  SDLoc sdl = getCurSDLoc();
4864  DebugLoc dl = getCurDebugLoc();
4865  SDValue Res;
4866 
4867  switch (Intrinsic) {
4868  default:
4869  // By default, turn this into a target intrinsic node.
4870  visitTargetIntrinsic(I, Intrinsic);
4871  return nullptr;
4872  case Intrinsic::vastart: visitVAStart(I); return nullptr;
4873  case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4874  case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4875  case Intrinsic::returnaddress:
4876  setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
4877  TLI.getPointerTy(DAG.getDataLayout()),
4878  getValue(I.getArgOperand(0))));
4879  return nullptr;
4880  case Intrinsic::addressofreturnaddress:
4881  setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
4882  TLI.getPointerTy(DAG.getDataLayout())));
4883  return nullptr;
4884  case Intrinsic::frameaddress:
4885  setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
4886  TLI.getPointerTy(DAG.getDataLayout()),
4887  getValue(I.getArgOperand(0))));
4888  return nullptr;
4889  case Intrinsic::read_register: {
4890  Value *Reg = I.getArgOperand(0);
4891  SDValue Chain = getRoot();
4892  SDValue RegName =
4893  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4894  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4895  Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4896  DAG.getVTList(VT, MVT::Other), Chain, RegName);
4897  setValue(&I, Res);
4898  DAG.setRoot(Res.getValue(1));
4899  return nullptr;
4900  }
4901  case Intrinsic::write_register: {
4902  Value *Reg = I.getArgOperand(0);
4903  Value *RegValue = I.getArgOperand(1);
4904  SDValue Chain = getRoot();
4905  SDValue RegName =
4906  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4907  DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
4908  RegName, getValue(RegValue)));
4909  return nullptr;
4910  }
4911  case Intrinsic::setjmp:
4912  return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4913  case Intrinsic::longjmp:
4914  return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4915  case Intrinsic::memcpy: {
4916  SDValue Op1 = getValue(I.getArgOperand(0));
4917  SDValue Op2 = getValue(I.getArgOperand(1));
4918  SDValue Op3 = getValue(I.getArgOperand(2));
4919  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4920  if (!Align)
4921  Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4922  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4923  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4924  SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4925  false, isTC,
4928  updateDAGForMaybeTailCall(MC);
4929  return nullptr;
4930  }
4931  case Intrinsic::memset: {
4932  SDValue Op1 = getValue(I.getArgOperand(0));
4933  SDValue Op2 = getValue(I.getArgOperand(1));
4934  SDValue Op3 = getValue(I.getArgOperand(2));
4935  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4936  if (!Align)
4937  Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4938  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4939  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4940  SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4941  isTC, MachinePointerInfo(I.getArgOperand(0)));
4942  updateDAGForMaybeTailCall(MS);
4943  return nullptr;
4944  }
4945  case Intrinsic::memmove: {
4946  SDValue Op1 = getValue(I.getArgOperand(0));
4947  SDValue Op2 = getValue(I.getArgOperand(1));
4948  SDValue Op3 = getValue(I.getArgOperand(2));
4949  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4950  if (!Align)
4951  Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4952  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4953  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4954  SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4955  isTC, MachinePointerInfo(I.getArgOperand(0)),
4957  updateDAGForMaybeTailCall(MM);
4958  return nullptr;
4959  }
4960  case Intrinsic::memcpy_element_unordered_atomic: {
4962  cast<ElementUnorderedAtomicMemCpyInst>(I);
4963  SDValue Dst = getValue(MI.getRawDest());
4964  SDValue Src = getValue(MI.getRawSource());
4965  SDValue Length = getValue(MI.getLength());
4966 
4967  // Emit a library call.
4969  TargetLowering::ArgListEntry Entry;
4970  Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
4971  Entry.Node = Dst;
4972  Args.push_back(Entry);
4973 
4974  Entry.Node = Src;
4975  Args.push_back(Entry);
4976 
4977  Entry.Ty = MI.getLength()->