LLVM  4.0.0
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/Loads.h"
27 #include "llvm/CodeGen/Analysis.h"
28 #include "llvm/CodeGen/FastISel.h"
40 #include "llvm/CodeGen/StackMaps.h"
42 #include "llvm/IR/CallingConv.h"
43 #include "llvm/IR/ConstantRange.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DataLayout.h"
46 #include "llvm/IR/DebugInfo.h"
47 #include "llvm/IR/DerivedTypes.h"
48 #include "llvm/IR/Function.h"
50 #include "llvm/IR/GlobalVariable.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/Instructions.h"
53 #include "llvm/IR/IntrinsicInst.h"
54 #include "llvm/IR/Intrinsics.h"
55 #include "llvm/IR/LLVMContext.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Statepoint.h"
58 #include "llvm/MC/MCSymbol.h"
60 #include "llvm/Support/Debug.h"
70 #include <algorithm>
71 #include <utility>
72 using namespace llvm;
73 
74 #define DEBUG_TYPE "isel"
75 
76 /// LimitFloatPrecision - Generate low-precision inline sequences for
77 /// some float libcalls (6, 8 or 12 bits).
78 static unsigned LimitFloatPrecision;
79 
81 LimitFPPrecision("limit-float-precision",
82  cl::desc("Generate low-precision inline sequences "
83  "for some float libcalls"),
85  cl::init(0));
86 
87 static cl::opt<bool>
88 EnableFMFInDAG("enable-fmf-dag", cl::init(true), cl::Hidden,
89  cl::desc("Enable fast-math-flags for DAG nodes"));
90 
91 /// Minimum jump table density for normal functions.
92 static cl::opt<unsigned>
93 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
94  cl::desc("Minimum density for building a jump table in "
95  "a normal function"));
96 
97 /// Minimum jump table density for -Os or -Oz functions.
98 static cl::opt<unsigned>
99 OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden,
100  cl::desc("Minimum density for building a jump table in "
101  "an optsize function"));
102 
103 
104 // Limit the width of DAG chains. This is important in general to prevent
105 // DAG-based analysis from blowing up. For example, alias analysis and
106 // load clustering may not complete in reasonable time. It is difficult to
107 // recognize and avoid this situation within each individual analysis, and
108 // future analyses are likely to have the same behavior. Limiting DAG width is
109 // the safe approach and will be especially important with global DAGs.
110 //
111 // MaxParallelChains default is arbitrarily high to avoid affecting
112 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
113 // sequence over this should have been converted to llvm.memcpy by the
114 // frontend. It is easy to induce this behavior with .ll code such as:
115 // %buffer = alloca [4096 x i8]
116 // %data = load [4096 x i8]* %argPtr
117 // store [4096 x i8] %data, [4096 x i8]* %buffer
118 static const unsigned MaxParallelChains = 64;
119 
120 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
121  const SDValue *Parts, unsigned NumParts,
122  MVT PartVT, EVT ValueVT, const Value *V);
123 
124 /// getCopyFromParts - Create a value that contains the specified legal parts
125 /// combined into the value they represent. If the parts combine to a type
126 /// larger than ValueVT then AssertOp can be used to specify whether the extra
127 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
128 /// (ISD::AssertSext).
130  const SDValue *Parts, unsigned NumParts,
131  MVT PartVT, EVT ValueVT, const Value *V,
132  Optional<ISD::NodeType> AssertOp = None) {
133  if (ValueVT.isVector())
134  return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
135  PartVT, ValueVT, V);
136 
137  assert(NumParts > 0 && "No parts to assemble!");
138  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
139  SDValue Val = Parts[0];
140 
141  if (NumParts > 1) {
142  // Assemble the value from multiple parts.
143  if (ValueVT.isInteger()) {
144  unsigned PartBits = PartVT.getSizeInBits();
145  unsigned ValueBits = ValueVT.getSizeInBits();
146 
147  // Assemble the power of 2 part.
148  unsigned RoundParts = NumParts & (NumParts - 1) ?
149  1 << Log2_32(NumParts) : NumParts;
150  unsigned RoundBits = PartBits * RoundParts;
151  EVT RoundVT = RoundBits == ValueBits ?
152  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
153  SDValue Lo, Hi;
154 
155  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
156 
157  if (RoundParts > 2) {
158  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
159  PartVT, HalfVT, V);
160  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
161  RoundParts / 2, PartVT, HalfVT, V);
162  } else {
163  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
164  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
165  }
166 
167  if (DAG.getDataLayout().isBigEndian())
168  std::swap(Lo, Hi);
169 
170  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
171 
172  if (RoundParts < NumParts) {
173  // Assemble the trailing non-power-of-2 part.
174  unsigned OddParts = NumParts - RoundParts;
175  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
176  Hi = getCopyFromParts(DAG, DL,
177  Parts + RoundParts, OddParts, PartVT, OddVT, V);
178 
179  // Combine the round and odd parts.
180  Lo = Val;
181  if (DAG.getDataLayout().isBigEndian())
182  std::swap(Lo, Hi);
183  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
184  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
185  Hi =
186  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
187  DAG.getConstant(Lo.getValueSizeInBits(), DL,
188  TLI.getPointerTy(DAG.getDataLayout())));
189  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
190  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
191  }
192  } else if (PartVT.isFloatingPoint()) {
193  // FP split into multiple FP parts (for ppcf128)
194  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
195  "Unexpected split");
196  SDValue Lo, Hi;
197  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
198  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
199  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
200  std::swap(Lo, Hi);
201  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
202  } else {
203  // FP split into integer parts (soft fp)
204  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
205  !PartVT.isVector() && "Unexpected split");
206  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
207  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
208  }
209  }
210 
211  // There is now one part, held in Val. Correct it to match ValueVT.
212  // PartEVT is the type of the register class that holds the value.
213  // ValueVT is the type of the inline asm operation.
214  EVT PartEVT = Val.getValueType();
215 
216  if (PartEVT == ValueVT)
217  return Val;
218 
219  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
220  ValueVT.bitsLT(PartEVT)) {
221  // For an FP value in an integer part, we need to truncate to the right
222  // width first.
223  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
224  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
225  }
226 
227  // Handle types that have the same size.
228  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
229  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
230 
231  // Handle types with different sizes.
232  if (PartEVT.isInteger() && ValueVT.isInteger()) {
233  if (ValueVT.bitsLT(PartEVT)) {
234  // For a truncate, see if we have any information to
235  // indicate whether the truncated bits will always be
236  // zero or sign-extension.
237  if (AssertOp.hasValue())
238  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
239  DAG.getValueType(ValueVT));
240  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
241  }
242  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
243  }
244 
245  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
246  // FP_ROUND's are always exact here.
247  if (ValueVT.bitsLT(Val.getValueType()))
248  return DAG.getNode(
249  ISD::FP_ROUND, DL, ValueVT, Val,
250  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
251 
252  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
253  }
254 
255  llvm_unreachable("Unknown mismatch!");
256 }
257 
259  const Twine &ErrMsg) {
260  const Instruction *I = dyn_cast_or_null<Instruction>(V);
261  if (!V)
262  return Ctx.emitError(ErrMsg);
263 
264  const char *AsmError = ", possible invalid constraint for vector type";
265  if (const CallInst *CI = dyn_cast<CallInst>(I))
266  if (isa<InlineAsm>(CI->getCalledValue()))
267  return Ctx.emitError(I, ErrMsg + AsmError);
268 
269  return Ctx.emitError(I, ErrMsg);
270 }
271 
272 /// getCopyFromPartsVector - Create a value that contains the specified legal
273 /// parts combined into the value they represent. If the parts combine to a
274 /// type larger than ValueVT then AssertOp can be used to specify whether the
275 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
276 /// ValueVT (ISD::AssertSext).
278  const SDValue *Parts, unsigned NumParts,
279  MVT PartVT, EVT ValueVT, const Value *V) {
280  assert(ValueVT.isVector() && "Not a vector value");
281  assert(NumParts > 0 && "No parts to assemble!");
282  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
283  SDValue Val = Parts[0];
284 
285  // Handle a multi-element vector.
286  if (NumParts > 1) {
287  EVT IntermediateVT;
288  MVT RegisterVT;
289  unsigned NumIntermediates;
290  unsigned NumRegs =
291  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
292  NumIntermediates, RegisterVT);
293  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
294  NumParts = NumRegs; // Silence a compiler warning.
295  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
296  assert(RegisterVT.getSizeInBits() ==
297  Parts[0].getSimpleValueType().getSizeInBits() &&
298  "Part type sizes don't match!");
299 
300  // Assemble the parts into intermediate operands.
301  SmallVector<SDValue, 8> Ops(NumIntermediates);
302  if (NumIntermediates == NumParts) {
303  // If the register was not expanded, truncate or copy the value,
304  // as appropriate.
305  for (unsigned i = 0; i != NumParts; ++i)
306  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
307  PartVT, IntermediateVT, V);
308  } else if (NumParts > 0) {
309  // If the intermediate type was expanded, build the intermediate
310  // operands from the parts.
311  assert(NumParts % NumIntermediates == 0 &&
312  "Must expand into a divisible number of parts!");
313  unsigned Factor = NumParts / NumIntermediates;
314  for (unsigned i = 0; i != NumIntermediates; ++i)
315  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
316  PartVT, IntermediateVT, V);
317  }
318 
319  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
320  // intermediate operands.
321  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
323  DL, ValueVT, Ops);
324  }
325 
326  // There is now one part, held in Val. Correct it to match ValueVT.
327  EVT PartEVT = Val.getValueType();
328 
329  if (PartEVT == ValueVT)
330  return Val;
331 
332  if (PartEVT.isVector()) {
333  // If the element type of the source/dest vectors are the same, but the
334  // parts vector has more elements than the value vector, then we have a
335  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
336  // elements we want.
337  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
338  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
339  "Cannot narrow, it would be a lossy transformation");
340  return DAG.getNode(
341  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
342  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
343  }
344 
345  // Vector/Vector bitcast.
346  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
347  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
348 
349  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
350  "Cannot handle this kind of promotion");
351  // Promoted vector extract
352  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
353 
354  }
355 
356  // Trivial bitcast if the types are the same size and the destination
357  // vector type is legal.
358  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
359  TLI.isTypeLegal(ValueVT))
360  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
361 
362  // Handle cases such as i8 -> <1 x i1>
363  if (ValueVT.getVectorNumElements() != 1) {
365  "non-trivial scalar-to-vector conversion");
366  return DAG.getUNDEF(ValueVT);
367  }
368 
369  if (ValueVT.getVectorNumElements() == 1 &&
370  ValueVT.getVectorElementType() != PartEVT)
371  Val = DAG.getAnyExtOrTrunc(Val, DL, ValueVT.getScalarType());
372 
373  return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
374 }
375 
376 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
377  SDValue Val, SDValue *Parts, unsigned NumParts,
378  MVT PartVT, const Value *V);
379 
380 /// getCopyToParts - Create a series of nodes that contain the specified value
381 /// split into legal parts. If the parts contain more bits than Val, then, for
382 /// integers, ExtendKind can be used to specify how to generate the extra bits.
383 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
384  SDValue *Parts, unsigned NumParts, MVT PartVT,
385  const Value *V,
386  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
387  EVT ValueVT = Val.getValueType();
388 
389  // Handle the vector case separately.
390  if (ValueVT.isVector())
391  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
392 
393  unsigned PartBits = PartVT.getSizeInBits();
394  unsigned OrigNumParts = NumParts;
395  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
396  "Copying to an illegal type!");
397 
398  if (NumParts == 0)
399  return;
400 
401  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
402  EVT PartEVT = PartVT;
403  if (PartEVT == ValueVT) {
404  assert(NumParts == 1 && "No-op copy with multiple parts!");
405  Parts[0] = Val;
406  return;
407  }
408 
409  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
410  // If the parts cover more bits than the value has, promote the value.
411  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
412  assert(NumParts == 1 && "Do not know what to promote to!");
413  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
414  } else {
415  if (ValueVT.isFloatingPoint()) {
416  // FP values need to be bitcast, then extended if they are being put
417  // into a larger container.
418  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
419  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
420  }
421  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
422  ValueVT.isInteger() &&
423  "Unknown mismatch!");
424  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
425  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
426  if (PartVT == MVT::x86mmx)
427  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
428  }
429  } else if (PartBits == ValueVT.getSizeInBits()) {
430  // Different types of the same size.
431  assert(NumParts == 1 && PartEVT != ValueVT);
432  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
433  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
434  // If the parts cover less bits than value has, truncate the value.
435  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
436  ValueVT.isInteger() &&
437  "Unknown mismatch!");
438  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
439  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
440  if (PartVT == MVT::x86mmx)
441  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
442  }
443 
444  // The value may have changed - recompute ValueVT.
445  ValueVT = Val.getValueType();
446  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
447  "Failed to tile the value with PartVT!");
448 
449  if (NumParts == 1) {
450  if (PartEVT != ValueVT) {
452  "scalar-to-vector conversion failed");
453  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
454  }
455 
456  Parts[0] = Val;
457  return;
458  }
459 
460  // Expand the value into multiple parts.
461  if (NumParts & (NumParts - 1)) {
462  // The number of parts is not a power of 2. Split off and copy the tail.
463  assert(PartVT.isInteger() && ValueVT.isInteger() &&
464  "Do not know what to expand to!");
465  unsigned RoundParts = 1 << Log2_32(NumParts);
466  unsigned RoundBits = RoundParts * PartBits;
467  unsigned OddParts = NumParts - RoundParts;
468  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
469  DAG.getIntPtrConstant(RoundBits, DL));
470  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
471 
472  if (DAG.getDataLayout().isBigEndian())
473  // The odd parts were reversed by getCopyToParts - unreverse them.
474  std::reverse(Parts + RoundParts, Parts + NumParts);
475 
476  NumParts = RoundParts;
477  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
478  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
479  }
480 
481  // The number of parts is a power of 2. Repeatedly bisect the value using
482  // EXTRACT_ELEMENT.
483  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
485  ValueVT.getSizeInBits()),
486  Val);
487 
488  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
489  for (unsigned i = 0; i < NumParts; i += StepSize) {
490  unsigned ThisBits = StepSize * PartBits / 2;
491  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
492  SDValue &Part0 = Parts[i];
493  SDValue &Part1 = Parts[i+StepSize/2];
494 
495  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
496  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
497  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
498  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
499 
500  if (ThisBits == PartBits && ThisVT != PartVT) {
501  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
502  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
503  }
504  }
505  }
506 
507  if (DAG.getDataLayout().isBigEndian())
508  std::reverse(Parts, Parts + OrigNumParts);
509 }
510 
511 
512 /// getCopyToPartsVector - Create a series of nodes that contain the specified
513 /// value split into legal parts.
514 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
515  SDValue Val, SDValue *Parts, unsigned NumParts,
516  MVT PartVT, const Value *V) {
517  EVT ValueVT = Val.getValueType();
518  assert(ValueVT.isVector() && "Not a vector");
519  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
520 
521  if (NumParts == 1) {
522  EVT PartEVT = PartVT;
523  if (PartEVT == ValueVT) {
524  // Nothing to do.
525  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
526  // Bitconvert vector->vector case.
527  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
528  } else if (PartVT.isVector() &&
529  PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
530  PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
531  EVT ElementVT = PartVT.getVectorElementType();
532  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
533  // undef elements.
535  for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
536  Ops.push_back(DAG.getNode(
537  ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
538  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
539 
540  for (unsigned i = ValueVT.getVectorNumElements(),
541  e = PartVT.getVectorNumElements(); i != e; ++i)
542  Ops.push_back(DAG.getUNDEF(ElementVT));
543 
544  Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
545 
546  // FIXME: Use CONCAT for 2x -> 4x.
547 
548  //SDValue UndefElts = DAG.getUNDEF(VectorTy);
549  //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
550  } else if (PartVT.isVector() &&
551  PartEVT.getVectorElementType().bitsGE(
552  ValueVT.getVectorElementType()) &&
553  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
554 
555  // Promoted vector extract
556  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
557  } else{
558  // Vector -> scalar conversion.
559  assert(ValueVT.getVectorNumElements() == 1 &&
560  "Only trivial vector-to-scalar conversions should get here!");
561  Val = DAG.getNode(
562  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
563  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
564 
565  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
566  }
567 
568  Parts[0] = Val;
569  return;
570  }
571 
572  // Handle a multi-element vector.
573  EVT IntermediateVT;
574  MVT RegisterVT;
575  unsigned NumIntermediates;
576  unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
577  IntermediateVT,
578  NumIntermediates, RegisterVT);
579  unsigned NumElements = ValueVT.getVectorNumElements();
580 
581  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
582  NumParts = NumRegs; // Silence a compiler warning.
583  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
584 
585  // Split the vector into intermediate operands.
586  SmallVector<SDValue, 8> Ops(NumIntermediates);
587  for (unsigned i = 0; i != NumIntermediates; ++i) {
588  if (IntermediateVT.isVector())
589  Ops[i] =
590  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
591  DAG.getConstant(i * (NumElements / NumIntermediates), DL,
592  TLI.getVectorIdxTy(DAG.getDataLayout())));
593  else
594  Ops[i] = DAG.getNode(
595  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
596  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
597  }
598 
599  // Split the intermediate operands into legal parts.
600  if (NumParts == NumIntermediates) {
601  // If the register was not expanded, promote or copy the value,
602  // as appropriate.
603  for (unsigned i = 0; i != NumParts; ++i)
604  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
605  } else if (NumParts > 0) {
606  // If the intermediate type was expanded, split each the value into
607  // legal parts.
608  assert(NumIntermediates != 0 && "division by zero");
609  assert(NumParts % NumIntermediates == 0 &&
610  "Must expand into a divisible number of parts!");
611  unsigned Factor = NumParts / NumIntermediates;
612  for (unsigned i = 0; i != NumIntermediates; ++i)
613  getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
614  }
615 }
616 
618 
620  EVT valuevt)
621  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
622 
624  const DataLayout &DL, unsigned Reg, Type *Ty) {
625  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
626 
627  for (EVT ValueVT : ValueVTs) {
628  unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
629  MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
630  for (unsigned i = 0; i != NumRegs; ++i)
631  Regs.push_back(Reg + i);
632  RegVTs.push_back(RegisterVT);
633  Reg += NumRegs;
634  }
635 }
636 
637 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
638 /// this value and returns the result as a ValueVT value. This uses
639 /// Chain/Flag as the input and updates them for the output Chain/Flag.
640 /// If the Flag pointer is NULL, no flag is used.
642  FunctionLoweringInfo &FuncInfo,
643  const SDLoc &dl, SDValue &Chain,
644  SDValue *Flag, const Value *V) const {
645  // A Value with type {} or [0 x %t] needs no registers.
646  if (ValueVTs.empty())
647  return SDValue();
648 
649  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
650 
651  // Assemble the legal parts into the final values.
652  SmallVector<SDValue, 4> Values(ValueVTs.size());
654  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
655  // Copy the legal parts from the registers.
656  EVT ValueVT = ValueVTs[Value];
657  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
658  MVT RegisterVT = RegVTs[Value];
659 
660  Parts.resize(NumRegs);
661  for (unsigned i = 0; i != NumRegs; ++i) {
662  SDValue P;
663  if (!Flag) {
664  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
665  } else {
666  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
667  *Flag = P.getValue(2);
668  }
669 
670  Chain = P.getValue(1);
671  Parts[i] = P;
672 
673  // If the source register was virtual and if we know something about it,
674  // add an assert node.
676  !RegisterVT.isInteger() || RegisterVT.isVector())
677  continue;
678 
680  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
681  if (!LOI)
682  continue;
683 
684  unsigned RegSize = RegisterVT.getSizeInBits();
685  unsigned NumSignBits = LOI->NumSignBits;
686  unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
687 
688  if (NumZeroBits == RegSize) {
689  // The current value is a zero.
690  // Explicitly express that as it would be easier for
691  // optimizations to kick in.
692  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
693  continue;
694  }
695 
696  // FIXME: We capture more information than the dag can represent. For
697  // now, just use the tightest assertzext/assertsext possible.
698  bool isSExt = true;
699  EVT FromVT(MVT::Other);
700  if (NumSignBits == RegSize) {
701  isSExt = true; // ASSERT SEXT 1
702  FromVT = MVT::i1;
703  } else if (NumZeroBits >= RegSize - 1) {
704  isSExt = false; // ASSERT ZEXT 1
705  FromVT = MVT::i1;
706  } else if (NumSignBits > RegSize - 8) {
707  isSExt = true; // ASSERT SEXT 8
708  FromVT = MVT::i8;
709  } else if (NumZeroBits >= RegSize - 8) {
710  isSExt = false; // ASSERT ZEXT 8
711  FromVT = MVT::i8;
712  } else if (NumSignBits > RegSize - 16) {
713  isSExt = true; // ASSERT SEXT 16
714  FromVT = MVT::i16;
715  } else if (NumZeroBits >= RegSize - 16) {
716  isSExt = false; // ASSERT ZEXT 16
717  FromVT = MVT::i16;
718  } else if (NumSignBits > RegSize - 32) {
719  isSExt = true; // ASSERT SEXT 32
720  FromVT = MVT::i32;
721  } else if (NumZeroBits >= RegSize - 32) {
722  isSExt = false; // ASSERT ZEXT 32
723  FromVT = MVT::i32;
724  } else {
725  continue;
726  }
727  // Add an assertion node.
728  assert(FromVT != MVT::Other);
729  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
730  RegisterVT, P, DAG.getValueType(FromVT));
731  }
732 
733  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
734  NumRegs, RegisterVT, ValueVT, V);
735  Part += NumRegs;
736  Parts.clear();
737  }
738 
739  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
740 }
741 
742 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
743 /// specified value into the registers specified by this object. This uses
744 /// Chain/Flag as the input and updates them for the output Chain/Flag.
745 /// If the Flag pointer is NULL, no flag is used.
747  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
748  const Value *V,
749  ISD::NodeType PreferredExtendType) const {
750  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
751  ISD::NodeType ExtendKind = PreferredExtendType;
752 
753  // Get the list of the values's legal parts.
754  unsigned NumRegs = Regs.size();
755  SmallVector<SDValue, 8> Parts(NumRegs);
756  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
757  EVT ValueVT = ValueVTs[Value];
758  unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
759  MVT RegisterVT = RegVTs[Value];
760 
761  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
762  ExtendKind = ISD::ZERO_EXTEND;
763 
764  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
765  &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
766  Part += NumParts;
767  }
768 
769  // Copy the parts into the registers.
770  SmallVector<SDValue, 8> Chains(NumRegs);
771  for (unsigned i = 0; i != NumRegs; ++i) {
772  SDValue Part;
773  if (!Flag) {
774  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
775  } else {
776  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
777  *Flag = Part.getValue(1);
778  }
779 
780  Chains[i] = Part.getValue(0);
781  }
782 
783  if (NumRegs == 1 || Flag)
784  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
785  // flagged to it. That is the CopyToReg nodes and the user are considered
786  // a single scheduling unit. If we create a TokenFactor and return it as
787  // chain, then the TokenFactor is both a predecessor (operand) of the
788  // user as well as a successor (the TF operands are flagged to the user).
789  // c1, f1 = CopyToReg
790  // c2, f2 = CopyToReg
791  // c3 = TokenFactor c1, c2
792  // ...
793  // = op c3, ..., f2
794  Chain = Chains[NumRegs-1];
795  else
796  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
797 }
798 
799 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
800 /// operand list. This adds the code marker and includes the number of
801 /// values added into it.
802 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
803  unsigned MatchingIdx, const SDLoc &dl,
804  SelectionDAG &DAG,
805  std::vector<SDValue> &Ops) const {
806  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
807 
808  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
809  if (HasMatching)
810  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
811  else if (!Regs.empty() &&
813  // Put the register class of the virtual registers in the flag word. That
814  // way, later passes can recompute register class constraints for inline
815  // assembly as well as normal instructions.
816  // Don't do this for tied operands that can use the regclass information
817  // from the def.
819  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
820  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
821  }
822 
823  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
824  Ops.push_back(Res);
825 
826  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
827  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
828  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
829  MVT RegisterVT = RegVTs[Value];
830  for (unsigned i = 0; i != NumRegs; ++i) {
831  assert(Reg < Regs.size() && "Mismatch in # registers expected");
832  unsigned TheReg = Regs[Reg++];
833  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
834 
835  if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
836  // If we clobbered the stack pointer, MFI should know about it.
838  }
839  }
840  }
841 }
842 
844  const TargetLibraryInfo *li) {
845  AA = &aa;
846  GFI = gfi;
847  LibInfo = li;
848  DL = &DAG.getDataLayout();
849  Context = DAG.getContext();
850  LPadToCallSiteMap.clear();
851 }
852 
853 /// clear - Clear out the current SelectionDAG and the associated
854 /// state and prepare this SelectionDAGBuilder object to be used
855 /// for a new block. This doesn't clear out information about
856 /// additional blocks that are needed to complete switch lowering
857 /// or PHI node updating; that information is cleared out as it is
858 /// consumed.
860  NodeMap.clear();
861  UnusedArgNodeMap.clear();
862  PendingLoads.clear();
863  PendingExports.clear();
864  CurInst = nullptr;
865  HasTailCall = false;
866  SDNodeOrder = LowestSDNodeOrder;
868 }
869 
870 /// clearDanglingDebugInfo - Clear the dangling debug information
871 /// map. This function is separated from the clear so that debug
872 /// information that is dangling in a basic block can be properly
873 /// resolved in a different basic block. This allows the
874 /// SelectionDAG to resolve dangling debug information attached
875 /// to PHI nodes.
877  DanglingDebugInfoMap.clear();
878 }
879 
880 /// getRoot - Return the current virtual root of the Selection DAG,
881 /// flushing any PendingLoad items. This must be done before emitting
882 /// a store or any other node that may need to be ordered after any
883 /// prior load instructions.
884 ///
886  if (PendingLoads.empty())
887  return DAG.getRoot();
888 
889  if (PendingLoads.size() == 1) {
890  SDValue Root = PendingLoads[0];
891  DAG.setRoot(Root);
892  PendingLoads.clear();
893  return Root;
894  }
895 
896  // Otherwise, we have to make a token factor node.
898  PendingLoads);
899  PendingLoads.clear();
900  DAG.setRoot(Root);
901  return Root;
902 }
903 
904 /// getControlRoot - Similar to getRoot, but instead of flushing all the
905 /// PendingLoad items, flush all the PendingExports items. It is necessary
906 /// to do this before emitting a terminator instruction.
907 ///
909  SDValue Root = DAG.getRoot();
910 
911  if (PendingExports.empty())
912  return Root;
913 
914  // Turn all of the CopyToReg chains into one factored node.
915  if (Root.getOpcode() != ISD::EntryToken) {
916  unsigned i = 0, e = PendingExports.size();
917  for (; i != e; ++i) {
918  assert(PendingExports[i].getNode()->getNumOperands() > 1);
919  if (PendingExports[i].getNode()->getOperand(0) == Root)
920  break; // Don't add the root if we already indirectly depend on it.
921  }
922 
923  if (i == e)
924  PendingExports.push_back(Root);
925  }
926 
928  PendingExports);
929  PendingExports.clear();
930  DAG.setRoot(Root);
931  return Root;
932 }
933 
935  // Set up outgoing PHI node register values before emitting the terminator.
936  if (isa<TerminatorInst>(&I)) {
937  HandlePHINodesInSuccessorBlocks(I.getParent());
938  }
939 
940  ++SDNodeOrder;
941 
942  CurInst = &I;
943 
944  visit(I.getOpcode(), I);
945 
946  if (!isa<TerminatorInst>(&I) && !HasTailCall &&
947  !isStatepoint(&I)) // statepoints handle their exports internally
949 
950  CurInst = nullptr;
951 }
952 
953 void SelectionDAGBuilder::visitPHI(const PHINode &) {
954  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
955 }
956 
957 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
958  // Note: this doesn't use InstVisitor, because it has to work with
959  // ConstantExpr's in addition to instructions.
960  switch (Opcode) {
961  default: llvm_unreachable("Unknown instruction type encountered!");
962  // Build the switch statement using the Instruction.def file.
963 #define HANDLE_INST(NUM, OPCODE, CLASS) \
964  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
965 #include "llvm/IR/Instruction.def"
966  }
967 }
968 
969 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
970 // generate the debug data structures now that we've seen its definition.
972  SDValue Val) {
973  DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
974  if (DDI.getDI()) {
975  const DbgValueInst *DI = DDI.getDI();
976  DebugLoc dl = DDI.getdl();
977  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
978  DILocalVariable *Variable = DI->getVariable();
979  DIExpression *Expr = DI->getExpression();
980  assert(Variable->isValidLocationForIntrinsic(dl) &&
981  "Expected inlined-at fields to agree");
982  uint64_t Offset = DI->getOffset();
983  SDDbgValue *SDV;
984  if (Val.getNode()) {
985  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
986  Val)) {
987  SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
988  DAG.AddDbgValue(SDV, Val.getNode(), false);
989  }
990  } else
991  DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
992  DanglingDebugInfoMap[V] = DanglingDebugInfo();
993  }
994 }
995 
996 /// getCopyFromRegs - If there was virtual register allocated for the value V
997 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1000  SDValue Result;
1001 
1002  if (It != FuncInfo.ValueMap.end()) {
1003  unsigned InReg = It->second;
1005  DAG.getDataLayout(), InReg, Ty);
1006  SDValue Chain = DAG.getEntryNode();
1007  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1008  resolveDanglingDebugInfo(V, Result);
1009  }
1010 
1011  return Result;
1012 }
1013 
1014 /// getValue - Return an SDValue for the given Value.
1016  // If we already have an SDValue for this value, use it. It's important
1017  // to do this first, so that we don't create a CopyFromReg if we already
1018  // have a regular SDValue.
1019  SDValue &N = NodeMap[V];
1020  if (N.getNode()) return N;
1021 
1022  // If there's a virtual register allocated and initialized for this
1023  // value, use it.
1024  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1025  return copyFromReg;
1026 
1027  // Otherwise create a new SDValue and remember it.
1028  SDValue Val = getValueImpl(V);
1029  NodeMap[V] = Val;
1030  resolveDanglingDebugInfo(V, Val);
1031  return Val;
1032 }
1033 
1034 // Return true if SDValue exists for the given Value
1036  return (NodeMap.find(V) != NodeMap.end()) ||
1038 }
1039 
1040 /// getNonRegisterValue - Return an SDValue for the given Value, but
1041 /// don't look in FuncInfo.ValueMap for a virtual register.
1043  // If we already have an SDValue for this value, use it.
1044  SDValue &N = NodeMap[V];
1045  if (N.getNode()) {
1046  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1047  // Remove the debug location from the node as the node is about to be used
1048  // in a location which may differ from the original debug location. This
1049  // is relevant to Constant and ConstantFP nodes because they can appear
1050  // as constant expressions inside PHI nodes.
1051  N->setDebugLoc(DebugLoc());
1052  }
1053  return N;
1054  }
1055 
1056  // Otherwise create a new SDValue and remember it.
1057  SDValue Val = getValueImpl(V);
1058  NodeMap[V] = Val;
1059  resolveDanglingDebugInfo(V, Val);
1060  return Val;
1061 }
1062 
1063 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1064 /// Create an SDValue for the given value.
1066  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1067 
1068  if (const Constant *C = dyn_cast<Constant>(V)) {
1069  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1070 
1071  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1072  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1073 
1074  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1075  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1076 
1077  if (isa<ConstantPointerNull>(C)) {
1078  unsigned AS = V->getType()->getPointerAddressSpace();
1079  return DAG.getConstant(0, getCurSDLoc(),
1080  TLI.getPointerTy(DAG.getDataLayout(), AS));
1081  }
1082 
1083  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1084  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1085 
1086  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1087  return DAG.getUNDEF(VT);
1088 
1089  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1090  visit(CE->getOpcode(), *CE);
1091  SDValue N1 = NodeMap[V];
1092  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1093  return N1;
1094  }
1095 
1096  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1098  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1099  OI != OE; ++OI) {
1100  SDNode *Val = getValue(*OI).getNode();
1101  // If the operand is an empty aggregate, there are no values.
1102  if (!Val) continue;
1103  // Add each leaf value from the operand to the Constants list
1104  // to form a flattened list of all the values.
1105  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1106  Constants.push_back(SDValue(Val, i));
1107  }
1108 
1109  return DAG.getMergeValues(Constants, getCurSDLoc());
1110  }
1111 
1112  if (const ConstantDataSequential *CDS =
1113  dyn_cast<ConstantDataSequential>(C)) {
1115  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1116  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1117  // Add each leaf value from the operand to the Constants list
1118  // to form a flattened list of all the values.
1119  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1120  Ops.push_back(SDValue(Val, i));
1121  }
1122 
1123  if (isa<ArrayType>(CDS->getType()))
1124  return DAG.getMergeValues(Ops, getCurSDLoc());
1125  return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
1126  VT, Ops);
1127  }
1128 
1129  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1130  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1131  "Unknown struct or array constant!");
1132 
1133  SmallVector<EVT, 4> ValueVTs;
1134  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1135  unsigned NumElts = ValueVTs.size();
1136  if (NumElts == 0)
1137  return SDValue(); // empty struct
1139  for (unsigned i = 0; i != NumElts; ++i) {
1140  EVT EltVT = ValueVTs[i];
1141  if (isa<UndefValue>(C))
1142  Constants[i] = DAG.getUNDEF(EltVT);
1143  else if (EltVT.isFloatingPoint())
1144  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1145  else
1146  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1147  }
1148 
1149  return DAG.getMergeValues(Constants, getCurSDLoc());
1150  }
1151 
1152  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1153  return DAG.getBlockAddress(BA, VT);
1154 
1155  VectorType *VecTy = cast<VectorType>(V->getType());
1156  unsigned NumElements = VecTy->getNumElements();
1157 
1158  // Now that we know the number and type of the elements, get that number of
1159  // elements into the Ops array based on what kind of constant it is.
1161  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1162  for (unsigned i = 0; i != NumElements; ++i)
1163  Ops.push_back(getValue(CV->getOperand(i)));
1164  } else {
1165  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1166  EVT EltVT =
1167  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1168 
1169  SDValue Op;
1170  if (EltVT.isFloatingPoint())
1171  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1172  else
1173  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1174  Ops.assign(NumElements, Op);
1175  }
1176 
1177  // Create a BUILD_VECTOR node.
1178  return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
1179  }
1180 
1181  // If this is a static alloca, generate it as the frameindex instead of
1182  // computation.
1183  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1185  FuncInfo.StaticAllocaMap.find(AI);
1186  if (SI != FuncInfo.StaticAllocaMap.end())
1187  return DAG.getFrameIndex(SI->second,
1188  TLI.getPointerTy(DAG.getDataLayout()));
1189  }
1190 
1191  // If this is an instruction which fast-isel has deferred, select it now.
1192  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1193  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1194  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1195  Inst->getType());
1196  SDValue Chain = DAG.getEntryNode();
1197  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1198  }
1199 
1200  llvm_unreachable("Can't get register for value!");
1201 }
1202 
1203 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1205  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1206  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1207  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1208  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1209  if (IsMSVCCXX || IsCoreCLR)
1210  CatchPadMBB->setIsEHFuncletEntry();
1211 
1213 }
1214 
1215 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1216  // Update machine-CFG edge.
1217  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1218  FuncInfo.MBB->addSuccessor(TargetMBB);
1219 
1221  bool IsSEH = isAsynchronousEHPersonality(Pers);
1222  if (IsSEH) {
1223  // If this is not a fall-through branch or optimizations are switched off,
1224  // emit the branch.
1225  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1226  TM.getOptLevel() == CodeGenOpt::None)
1228  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1229  return;
1230  }
1231 
1232  // Figure out the funclet membership for the catchret's successor.
1233  // This will be used by the FuncletLayout pass to determine how to order the
1234  // BB's.
1235  // A 'catchret' returns to the outer scope's color.
1236  Value *ParentPad = I.getCatchSwitchParentPad();
1237  const BasicBlock *SuccessorColor;
1238  if (isa<ConstantTokenNone>(ParentPad))
1239  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1240  else
1241  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1242  assert(SuccessorColor && "No parent funclet for catchret!");
1243  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1244  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1245 
1246  // Create the terminator node.
1248  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1249  DAG.getBasicBlock(SuccessorColorMBB));
1250  DAG.setRoot(Ret);
1251 }
1252 
1253 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1254  // Don't emit any special code for the cleanuppad instruction. It just marks
1255  // the start of a funclet.
1258 }
1259 
1260 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1261 /// many places it could ultimately go. In the IR, we have a single unwind
1262 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1263 /// This function skips over imaginary basic blocks that hold catchswitch
1264 /// instructions, and finds all the "real" machine
1265 /// basic block destinations. As those destinations may not be successors of
1266 /// EHPadBB, here we also calculate the edge probability to those destinations.
1267 /// The passed-in Prob is the edge probability to EHPadBB.
1269  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1270  BranchProbability Prob,
1271  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1272  &UnwindDests) {
1273  EHPersonality Personality =
1275  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1276  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1277 
1278  while (EHPadBB) {
1279  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1280  BasicBlock *NewEHPadBB = nullptr;
1281  if (isa<LandingPadInst>(Pad)) {
1282  // Stop on landingpads. They are not funclets.
1283  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1284  break;
1285  } else if (isa<CleanupPadInst>(Pad)) {
1286  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1287  // personalities.
1288  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1289  UnwindDests.back().first->setIsEHFuncletEntry();
1290  break;
1291  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1292  // Add the catchpad handlers to the possible destinations.
1293  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1294  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1295  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1296  if (IsMSVCCXX || IsCoreCLR)
1297  UnwindDests.back().first->setIsEHFuncletEntry();
1298  }
1299  NewEHPadBB = CatchSwitch->getUnwindDest();
1300  } else {
1301  continue;
1302  }
1303 
1304  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1305  if (BPI && NewEHPadBB)
1306  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1307  EHPadBB = NewEHPadBB;
1308  }
1309 }
1310 
1311 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1312  // Update successor info.
1314  auto UnwindDest = I.getUnwindDest();
1316  BranchProbability UnwindDestProb =
1317  (BPI && UnwindDest)
1318  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1320  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1321  for (auto &UnwindDest : UnwindDests) {
1322  UnwindDest.first->setIsEHPad();
1323  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1324  }
1326 
1327  // Create the terminator node.
1328  SDValue Ret =
1330  DAG.setRoot(Ret);
1331 }
1332 
1333 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1334  report_fatal_error("visitCatchSwitch not yet implemented!");
1335 }
1336 
1337 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1338  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1339  auto &DL = DAG.getDataLayout();
1340  SDValue Chain = getControlRoot();
1342  SmallVector<SDValue, 8> OutVals;
1343 
1344  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1345  // lower
1346  //
1347  // %val = call <ty> @llvm.experimental.deoptimize()
1348  // ret <ty> %val
1349  //
1350  // differently.
1353  return;
1354  }
1355 
1356  if (!FuncInfo.CanLowerReturn) {
1357  unsigned DemoteReg = FuncInfo.DemoteRegister;
1358  const Function *F = I.getParent()->getParent();
1359 
1360  // Emit a store of the return value through the virtual register.
1361  // Leave Outs empty so that LowerReturn won't try to load return
1362  // registers the usual way.
1363  SmallVector<EVT, 1> PtrValueVTs;
1365  PtrValueVTs);
1366 
1368  DemoteReg, PtrValueVTs[0]);
1369  SDValue RetOp = getValue(I.getOperand(0));
1370 
1371  SmallVector<EVT, 4> ValueVTs;
1373  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1374  unsigned NumValues = ValueVTs.size();
1375 
1376  // An aggregate return value cannot wrap around the address space, so
1377  // offsets to its parts don't wrap either.
1379  Flags.setNoUnsignedWrap(true);
1380 
1381  SmallVector<SDValue, 4> Chains(NumValues);
1382  for (unsigned i = 0; i != NumValues; ++i) {
1384  RetPtr.getValueType(), RetPtr,
1385  DAG.getIntPtrConstant(Offsets[i],
1386  getCurSDLoc()),
1387  &Flags);
1388  Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1389  SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1390  // FIXME: better loc info would be nice.
1391  Add, MachinePointerInfo());
1392  }
1393 
1395  MVT::Other, Chains);
1396  } else if (I.getNumOperands() != 0) {
1397  SmallVector<EVT, 4> ValueVTs;
1398  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1399  unsigned NumValues = ValueVTs.size();
1400  if (NumValues) {
1401  SDValue RetOp = getValue(I.getOperand(0));
1402 
1403  const Function *F = I.getParent()->getParent();
1404 
1405  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1406  if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1407  Attribute::SExt))
1408  ExtendKind = ISD::SIGN_EXTEND;
1409  else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1410  Attribute::ZExt))
1411  ExtendKind = ISD::ZERO_EXTEND;
1412 
1413  LLVMContext &Context = F->getContext();
1414  bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1415  Attribute::InReg);
1416 
1417  for (unsigned j = 0; j != NumValues; ++j) {
1418  EVT VT = ValueVTs[j];
1419 
1420  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1421  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1422 
1423  unsigned NumParts = TLI.getNumRegisters(Context, VT);
1424  MVT PartVT = TLI.getRegisterType(Context, VT);
1425  SmallVector<SDValue, 4> Parts(NumParts);
1427  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1428  &Parts[0], NumParts, PartVT, &I, ExtendKind);
1429 
1430  // 'inreg' on function refers to return value
1431  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1432  if (RetInReg)
1433  Flags.setInReg();
1434 
1435  // Propagate extension type if any
1436  if (ExtendKind == ISD::SIGN_EXTEND)
1437  Flags.setSExt();
1438  else if (ExtendKind == ISD::ZERO_EXTEND)
1439  Flags.setZExt();
1440 
1441  for (unsigned i = 0; i < NumParts; ++i) {
1442  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1443  VT, /*isfixed=*/true, 0, 0));
1444  OutVals.push_back(Parts[i]);
1445  }
1446  }
1447  }
1448  }
1449 
1450  // Push in swifterror virtual register as the last element of Outs. This makes
1451  // sure swifterror virtual register will be returned in the swifterror
1452  // physical register.
1453  const Function *F = I.getParent()->getParent();
1454  if (TLI.supportSwiftError() &&
1455  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1456  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1457  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1458  Flags.setSwiftError();
1459  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1460  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1461  true /*isfixed*/, 1 /*origidx*/,
1462  0 /*partOffs*/));
1463  // Create SDNode for the swifterror virtual register.
1466  EVT(TLI.getPointerTy(DL))));
1467  }
1468 
1469  bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1470  CallingConv::ID CallConv =
1473  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1474 
1475  // Verify that the target's LowerReturn behaved as expected.
1476  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1477  "LowerReturn didn't return a valid chain!");
1478 
1479  // Update the DAG with the new chain value resulting from return lowering.
1480  DAG.setRoot(Chain);
1481 }
1482 
1483 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1484 /// created for it, emit nodes to copy the value into the virtual
1485 /// registers.
1487  // Skip empty types
1488  if (V->getType()->isEmptyTy())
1489  return;
1490 
1492  if (VMI != FuncInfo.ValueMap.end()) {
1493  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1494  CopyValueToVirtualRegister(V, VMI->second);
1495  }
1496 }
1497 
1498 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1499 /// the current basic block, add it to ValueMap now so that we'll get a
1500 /// CopyTo/FromReg.
1502  // No need to export constants.
1503  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1504 
1505  // Already exported?
1506  if (FuncInfo.isExportedInst(V)) return;
1507 
1508  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1510 }
1511 
1513  const BasicBlock *FromBB) {
1514  // The operands of the setcc have to be in this block. We don't know
1515  // how to export them from some other block.
1516  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1517  // Can export from current BB.
1518  if (VI->getParent() == FromBB)
1519  return true;
1520 
1521  // Is already exported, noop.
1522  return FuncInfo.isExportedInst(V);
1523  }
1524 
1525  // If this is an argument, we can export it if the BB is the entry block or
1526  // if it is already exported.
1527  if (isa<Argument>(V)) {
1528  if (FromBB == &FromBB->getParent()->getEntryBlock())
1529  return true;
1530 
1531  // Otherwise, can only export this if it is already exported.
1532  return FuncInfo.isExportedInst(V);
1533  }
1534 
1535  // Otherwise, constants can always be exported.
1536  return true;
1537 }
1538 
1539 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1541 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1542  const MachineBasicBlock *Dst) const {
1544  const BasicBlock *SrcBB = Src->getBasicBlock();
1545  const BasicBlock *DstBB = Dst->getBasicBlock();
1546  if (!BPI) {
1547  // If BPI is not available, set the default probability as 1 / N, where N is
1548  // the number of successors.
1549  auto SuccSize = std::max<uint32_t>(
1550  std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1551  return BranchProbability(1, SuccSize);
1552  }
1553  return BPI->getEdgeProbability(SrcBB, DstBB);
1554 }
1555 
1556 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1557  MachineBasicBlock *Dst,
1558  BranchProbability Prob) {
1559  if (!FuncInfo.BPI)
1560  Src->addSuccessorWithoutProb(Dst);
1561  else {
1562  if (Prob.isUnknown())
1563  Prob = getEdgeProbability(Src, Dst);
1564  Src->addSuccessor(Dst, Prob);
1565  }
1566 }
1567 
1568 static bool InBlock(const Value *V, const BasicBlock *BB) {
1569  if (const Instruction *I = dyn_cast<Instruction>(V))
1570  return I->getParent() == BB;
1571  return true;
1572 }
1573 
1574 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1575 /// This function emits a branch and is used at the leaves of an OR or an
1576 /// AND operator tree.
1577 ///
1578 void
1580  MachineBasicBlock *TBB,
1581  MachineBasicBlock *FBB,
1582  MachineBasicBlock *CurBB,
1583  MachineBasicBlock *SwitchBB,
1584  BranchProbability TProb,
1585  BranchProbability FProb) {
1586  const BasicBlock *BB = CurBB->getBasicBlock();
1587 
1588  // If the leaf of the tree is a comparison, merge the condition into
1589  // the caseblock.
1590  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1591  // The operands of the cmp have to be in this block. We don't know
1592  // how to export them from some other block. If this is the first block
1593  // of the sequence, no exporting is needed.
1594  if (CurBB == SwitchBB ||
1595  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1596  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1597  ISD::CondCode Condition;
1598  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1599  Condition = getICmpCondCode(IC->getPredicate());
1600  } else {
1601  const FCmpInst *FC = cast<FCmpInst>(Cond);
1602  Condition = getFCmpCondCode(FC->getPredicate());
1603  if (TM.Options.NoNaNsFPMath)
1604  Condition = getFCmpCodeWithoutNaN(Condition);
1605  }
1606 
1607  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1608  TBB, FBB, CurBB, TProb, FProb);
1609  SwitchCases.push_back(CB);
1610  return;
1611  }
1612  }
1613 
1614  // Create a CaseBlock record representing this branch.
1615  CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
1616  nullptr, TBB, FBB, CurBB, TProb, FProb);
1617  SwitchCases.push_back(CB);
1618 }
1619 
1620 /// FindMergedConditions - If Cond is an expression like
1622  MachineBasicBlock *TBB,
1623  MachineBasicBlock *FBB,
1624  MachineBasicBlock *CurBB,
1625  MachineBasicBlock *SwitchBB,
1627  BranchProbability TProb,
1628  BranchProbability FProb) {
1629  // If this node is not part of the or/and tree, emit it as a branch.
1630  const Instruction *BOp = dyn_cast<Instruction>(Cond);
1631  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1632  (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
1633  BOp->getParent() != CurBB->getBasicBlock() ||
1634  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1635  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1636  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1637  TProb, FProb);
1638  return;
1639  }
1640 
1641  // Create TmpBB after CurBB.
1642  MachineFunction::iterator BBI(CurBB);
1645  CurBB->getParent()->insert(++BBI, TmpBB);
1646 
1647  if (Opc == Instruction::Or) {
1648  // Codegen X | Y as:
1649  // BB1:
1650  // jmp_if_X TBB
1651  // jmp TmpBB
1652  // TmpBB:
1653  // jmp_if_Y TBB
1654  // jmp FBB
1655  //
1656 
1657  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1658  // The requirement is that
1659  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1660  // = TrueProb for original BB.
1661  // Assuming the original probabilities are A and B, one choice is to set
1662  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1663  // A/(1+B) and 2B/(1+B). This choice assumes that
1664  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1665  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1666  // TmpBB, but the math is more complicated.
1667 
1668  auto NewTrueProb = TProb / 2;
1669  auto NewFalseProb = TProb / 2 + FProb;
1670  // Emit the LHS condition.
1671  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1672  NewTrueProb, NewFalseProb);
1673 
1674  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1675  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1676  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1677  // Emit the RHS condition into TmpBB.
1678  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1679  Probs[0], Probs[1]);
1680  } else {
1681  assert(Opc == Instruction::And && "Unknown merge op!");
1682  // Codegen X & Y as:
1683  // BB1:
1684  // jmp_if_X TmpBB
1685  // jmp FBB
1686  // TmpBB:
1687  // jmp_if_Y TBB
1688  // jmp FBB
1689  //
1690  // This requires creation of TmpBB after CurBB.
1691 
1692  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1693  // The requirement is that
1694  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1695  // = FalseProb for original BB.
1696  // Assuming the original probabilities are A and B, one choice is to set
1697  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1698  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1699  // TrueProb for BB1 * FalseProb for TmpBB.
1700 
1701  auto NewTrueProb = TProb + FProb / 2;
1702  auto NewFalseProb = FProb / 2;
1703  // Emit the LHS condition.
1704  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1705  NewTrueProb, NewFalseProb);
1706 
1707  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1708  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1709  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1710  // Emit the RHS condition into TmpBB.
1711  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1712  Probs[0], Probs[1]);
1713  }
1714 }
1715 
1716 /// If the set of cases should be emitted as a series of branches, return true.
1717 /// If we should emit this as a bunch of and/or'd together conditions, return
1718 /// false.
1719 bool
1720 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1721  if (Cases.size() != 2) return true;
1722 
1723  // If this is two comparisons of the same values or'd or and'd together, they
1724  // will get folded into a single comparison, so don't emit two blocks.
1725  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1726  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1727  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1728  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1729  return false;
1730  }
1731 
1732  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1733  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1734  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1735  Cases[0].CC == Cases[1].CC &&
1736  isa<Constant>(Cases[0].CmpRHS) &&
1737  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1738  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1739  return false;
1740  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1741  return false;
1742  }
1743 
1744  return true;
1745 }
1746 
1747 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1748  MachineBasicBlock *BrMBB = FuncInfo.MBB;
1749 
1750  // Update machine-CFG edges.
1751  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1752 
1753  if (I.isUnconditional()) {
1754  // Update machine-CFG edges.
1755  BrMBB->addSuccessor(Succ0MBB);
1756 
1757  // If this is not a fall-through branch or optimizations are switched off,
1758  // emit the branch.
1759  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1762  DAG.getBasicBlock(Succ0MBB)));
1763 
1764  return;
1765  }
1766 
1767  // If this condition is one of the special cases we handle, do special stuff
1768  // now.
1769  const Value *CondVal = I.getCondition();
1770  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1771 
1772  // If this is a series of conditions that are or'd or and'd together, emit
1773  // this as a sequence of branches instead of setcc's with and/or operations.
1774  // As long as jumps are not expensive, this should improve performance.
1775  // For example, instead of something like:
1776  // cmp A, B
1777  // C = seteq
1778  // cmp D, E
1779  // F = setle
1780  // or C, F
1781  // jnz foo
1782  // Emit:
1783  // cmp A, B
1784  // je foo
1785  // cmp D, E
1786  // jle foo
1787  //
1788  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1789  Instruction::BinaryOps Opcode = BOp->getOpcode();
1790  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1792  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1793  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1794  Opcode,
1795  getEdgeProbability(BrMBB, Succ0MBB),
1796  getEdgeProbability(BrMBB, Succ1MBB));
1797  // If the compares in later blocks need to use values not currently
1798  // exported from this block, export them now. This block should always
1799  // be the first entry.
1800  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
1801 
1802  // Allow some cases to be rejected.
1804  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1807  }
1808 
1809  // Emit the branch for this block.
1810  visitSwitchCase(SwitchCases[0], BrMBB);
1811  SwitchCases.erase(SwitchCases.begin());
1812  return;
1813  }
1814 
1815  // Okay, we decided not to do this, remove any inserted MBB's and clear
1816  // SwitchCases.
1817  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1818  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1819 
1820  SwitchCases.clear();
1821  }
1822  }
1823 
1824  // Create a CaseBlock record representing this branch.
1825  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1826  nullptr, Succ0MBB, Succ1MBB, BrMBB);
1827 
1828  // Use visitSwitchCase to actually insert the fast branch sequence for this
1829  // cond branch.
1830  visitSwitchCase(CB, BrMBB);
1831 }
1832 
1833 /// visitSwitchCase - Emits the necessary code to represent a single node in
1834 /// the binary search tree resulting from lowering a switch instruction.
1836  MachineBasicBlock *SwitchBB) {
1837  SDValue Cond;
1838  SDValue CondLHS = getValue(CB.CmpLHS);
1839  SDLoc dl = getCurSDLoc();
1840 
1841  // Build the setcc now.
1842  if (!CB.CmpMHS) {
1843  // Fold "(X == true)" to X and "(X == false)" to !X to
1844  // handle common cases produced by branch lowering.
1845  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1846  CB.CC == ISD::SETEQ)
1847  Cond = CondLHS;
1848  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1849  CB.CC == ISD::SETEQ) {
1850  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1851  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1852  } else
1853  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1854  } else {
1855  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
1856 
1857  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1858  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1859 
1860  SDValue CmpOp = getValue(CB.CmpMHS);
1861  EVT VT = CmpOp.getValueType();
1862 
1863  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1864  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1865  ISD::SETLE);
1866  } else {
1867  SDValue SUB = DAG.getNode(ISD::SUB, dl,
1868  VT, CmpOp, DAG.getConstant(Low, dl, VT));
1869  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1870  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1871  }
1872  }
1873 
1874  // Update successor info
1875  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1876  // TrueBB and FalseBB are always different unless the incoming IR is
1877  // degenerate. This only happens when running llc on weird IR.
1878  if (CB.TrueBB != CB.FalseBB)
1879  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1880  SwitchBB->normalizeSuccProbs();
1881 
1882  // If the lhs block is the next block, invert the condition so that we can
1883  // fall through to the lhs instead of the rhs block.
1884  if (CB.TrueBB == NextBlock(SwitchBB)) {
1885  std::swap(CB.TrueBB, CB.FalseBB);
1886  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1887  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1888  }
1889 
1890  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1891  MVT::Other, getControlRoot(), Cond,
1892  DAG.getBasicBlock(CB.TrueBB));
1893 
1894  // Insert the false branch. Do this even if it's a fall through branch,
1895  // this makes it easier to do DAG optimizations which require inverting
1896  // the branch condition.
1897  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1898  DAG.getBasicBlock(CB.FalseBB));
1899 
1900  DAG.setRoot(BrCond);
1901 }
1902 
1903 /// visitJumpTable - Emit JumpTable node in the current MBB
1905  // Emit the code for the jump table
1906  assert(JT.Reg != -1U && "Should lower JT Header first!");
1909  JT.Reg, PTy);
1910  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1911  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1912  MVT::Other, Index.getValue(1),
1913  Table, Index);
1914  DAG.setRoot(BrJumpTable);
1915 }
1916 
1917 /// visitJumpTableHeader - This function emits necessary code to produce index
1918 /// in the JumpTable from switch case.
1920  JumpTableHeader &JTH,
1921  MachineBasicBlock *SwitchBB) {
1922  SDLoc dl = getCurSDLoc();
1923 
1924  // Subtract the lowest switch case value from the value being switched on and
1925  // conditional branch to default mbb if the result is greater than the
1926  // difference between smallest and largest cases.
1927  SDValue SwitchOp = getValue(JTH.SValue);
1928  EVT VT = SwitchOp.getValueType();
1929  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1930  DAG.getConstant(JTH.First, dl, VT));
1931 
1932  // The SDNode we just created, which holds the value being switched on minus
1933  // the smallest case value, needs to be copied to a virtual register so it
1934  // can be used as an index into the jump table in a subsequent basic block.
1935  // This value may be smaller or larger than the target's pointer type, and
1936  // therefore require extension or truncating.
1937  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1938  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
1939 
1940  unsigned JumpTableReg =
1942  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
1943  JumpTableReg, SwitchOp);
1944  JT.Reg = JumpTableReg;
1945 
1946  // Emit the range check for the jump table, and branch to the default block
1947  // for the switch statement if the value being switched on exceeds the largest
1948  // case in the switch.
1949  SDValue CMP = DAG.getSetCC(
1951  Sub.getValueType()),
1952  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
1953 
1954  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1955  MVT::Other, CopyTo, CMP,
1956  DAG.getBasicBlock(JT.Default));
1957 
1958  // Avoid emitting unnecessary branches to the next block.
1959  if (JT.MBB != NextBlock(SwitchBB))
1960  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1961  DAG.getBasicBlock(JT.MBB));
1962 
1963  DAG.setRoot(BrCond);
1964 }
1965 
1966 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
1967 /// variable if there exists one.
1969  SDValue &Chain) {
1970  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1971  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1972  MachineFunction &MF = DAG.getMachineFunction();
1974  MachineSDNode *Node =
1975  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
1976  if (Global) {
1977  MachinePointerInfo MPInfo(Global);
1981  *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
1982  DAG.getEVTAlignment(PtrTy));
1983  Node->setMemRefs(MemRefs, MemRefs + 1);
1984  }
1985  return SDValue(Node, 0);
1986 }
1987 
1988 /// Codegen a new tail for a stack protector check ParentMBB which has had its
1989 /// tail spliced into a stack protector check success bb.
1990 ///
1991 /// For a high level explanation of how this fits into the stack protector
1992 /// generation see the comment on the declaration of class
1993 /// StackProtectorDescriptor.
1994 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
1995  MachineBasicBlock *ParentBB) {
1996 
1997  // First create the loads to the guard/stack slot for the comparison.
1998  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1999  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2000 
2001  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2002  int FI = MFI.getStackProtectorIndex();
2003 
2004  SDValue Guard;
2005  SDLoc dl = getCurSDLoc();
2006  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2007  const Module &M = *ParentBB->getParent()->getFunction()->getParent();
2008  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2009 
2010  // Generate code to load the content of the guard slot.
2011  SDValue StackSlot = DAG.getLoad(
2012  PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2015 
2016  // Retrieve guard check function, nullptr if instrumentation is inlined.
2017  if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2018  // The target provides a guard check function to validate the guard value.
2019  // Generate a call to that function with the content of the guard slot as
2020  // argument.
2021  auto *Fn = cast<Function>(GuardCheck);
2022  FunctionType *FnTy = Fn->getFunctionType();
2023  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2024 
2027  Entry.Node = StackSlot;
2028  Entry.Ty = FnTy->getParamType(0);
2029  if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2030  Entry.isInReg = true;
2031  Args.push_back(Entry);
2032 
2034  CLI.setDebugLoc(getCurSDLoc())
2036  .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2037  getValue(GuardCheck), std::move(Args));
2038 
2039  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2040  DAG.setRoot(Result.second);
2041  return;
2042  }
2043 
2044  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2045  // Otherwise, emit a volatile load to retrieve the stack guard value.
2046  SDValue Chain = DAG.getEntryNode();
2047  if (TLI.useLoadStackGuardNode()) {
2048  Guard = getLoadStackGuard(DAG, dl, Chain);
2049  } else {
2050  const Value *IRGuard = TLI.getSDagStackGuard(M);
2051  SDValue GuardPtr = getValue(IRGuard);
2052 
2053  Guard =
2054  DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2056  }
2057 
2058  // Perform the comparison via a subtract/getsetcc.
2059  EVT VT = Guard.getValueType();
2060  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2061 
2063  *DAG.getContext(),
2064  Sub.getValueType()),
2065  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2066 
2067  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2068  // branch to failure MBB.
2069  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2070  MVT::Other, StackSlot.getOperand(0),
2071  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2072  // Otherwise branch to success MBB.
2073  SDValue Br = DAG.getNode(ISD::BR, dl,
2074  MVT::Other, BrCond,
2075  DAG.getBasicBlock(SPD.getSuccessMBB()));
2076 
2077  DAG.setRoot(Br);
2078 }
2079 
2080 /// Codegen the failure basic block for a stack protector check.
2081 ///
2082 /// A failure stack protector machine basic block consists simply of a call to
2083 /// __stack_chk_fail().
2084 ///
2085 /// For a high level explanation of how this fits into the stack protector
2086 /// generation see the comment on the declaration of class
2087 /// StackProtectorDescriptor.
2088 void
2089 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2090  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2091  SDValue Chain =
2093  None, false, getCurSDLoc(), false, false).second;
2094  DAG.setRoot(Chain);
2095 }
2096 
2097 /// visitBitTestHeader - This function emits necessary code to produce value
2098 /// suitable for "bit tests"
2100  MachineBasicBlock *SwitchBB) {
2101  SDLoc dl = getCurSDLoc();
2102 
2103  // Subtract the minimum value
2104  SDValue SwitchOp = getValue(B.SValue);
2105  EVT VT = SwitchOp.getValueType();
2106  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2107  DAG.getConstant(B.First, dl, VT));
2108 
2109  // Check range
2110  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2111  SDValue RangeCmp = DAG.getSetCC(
2113  Sub.getValueType()),
2114  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2115 
2116  // Determine the type of the test operands.
2117  bool UsePtrType = false;
2118  if (!TLI.isTypeLegal(VT))
2119  UsePtrType = true;
2120  else {
2121  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2122  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2123  // Switch table case range are encoded into series of masks.
2124  // Just use pointer type, it's guaranteed to fit.
2125  UsePtrType = true;
2126  break;
2127  }
2128  }
2129  if (UsePtrType) {
2130  VT = TLI.getPointerTy(DAG.getDataLayout());
2131  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2132  }
2133 
2134  B.RegVT = VT.getSimpleVT();
2135  B.Reg = FuncInfo.CreateReg(B.RegVT);
2136  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2137 
2138  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2139 
2140  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2141  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2142  SwitchBB->normalizeSuccProbs();
2143 
2144  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2145  MVT::Other, CopyTo, RangeCmp,
2146  DAG.getBasicBlock(B.Default));
2147 
2148  // Avoid emitting unnecessary branches to the next block.
2149  if (MBB != NextBlock(SwitchBB))
2150  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2151  DAG.getBasicBlock(MBB));
2152 
2153  DAG.setRoot(BrRange);
2154 }
2155 
2156 /// visitBitTestCase - this function produces one "bit test"
2158  MachineBasicBlock* NextMBB,
2159  BranchProbability BranchProbToNext,
2160  unsigned Reg,
2161  BitTestCase &B,
2162  MachineBasicBlock *SwitchBB) {
2163  SDLoc dl = getCurSDLoc();
2164  MVT VT = BB.RegVT;
2165  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2166  SDValue Cmp;
2167  unsigned PopCount = countPopulation(B.Mask);
2168  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2169  if (PopCount == 1) {
2170  // Testing for a single bit; just compare the shift count with what it
2171  // would need to be to shift a 1 bit in that position.
2172  Cmp = DAG.getSetCC(
2173  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2174  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2175  ISD::SETEQ);
2176  } else if (PopCount == BB.Range) {
2177  // There is only one zero bit in the range, test for it directly.
2178  Cmp = DAG.getSetCC(
2179  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2180  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2181  ISD::SETNE);
2182  } else {
2183  // Make desired shift
2184  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2185  DAG.getConstant(1, dl, VT), ShiftOp);
2186 
2187  // Emit bit tests and jumps
2188  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2189  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2190  Cmp = DAG.getSetCC(
2191  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2192  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2193  }
2194 
2195  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2196  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2197  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2198  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2199  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2200  // one as they are relative probabilities (and thus work more like weights),
2201  // and hence we need to normalize them to let the sum of them become one.
2202  SwitchBB->normalizeSuccProbs();
2203 
2204  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2206  Cmp, DAG.getBasicBlock(B.TargetBB));
2207 
2208  // Avoid emitting unnecessary branches to the next block.
2209  if (NextMBB != NextBlock(SwitchBB))
2210  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2211  DAG.getBasicBlock(NextMBB));
2212 
2213  DAG.setRoot(BrAnd);
2214 }
2215 
2216 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2217  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2218 
2219  // Retrieve successors. Look through artificial IR level blocks like
2220  // catchswitch for successors.
2222  const BasicBlock *EHPadBB = I.getSuccessor(1);
2223 
2224  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2225  // have to do anything here to lower funclet bundles.
2228  "Cannot lower invokes with arbitrary operand bundles yet!");
2229 
2230  const Value *Callee(I.getCalledValue());
2231  const Function *Fn = dyn_cast<Function>(Callee);
2232  if (isa<InlineAsm>(Callee))
2233  visitInlineAsm(&I);
2234  else if (Fn && Fn->isIntrinsic()) {
2235  switch (Fn->getIntrinsicID()) {
2236  default:
2237  llvm_unreachable("Cannot invoke this intrinsic");
2238  case Intrinsic::donothing:
2239  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2240  break;
2241  case Intrinsic::experimental_patchpoint_void:
2242  case Intrinsic::experimental_patchpoint_i64:
2243  visitPatchpoint(&I, EHPadBB);
2244  break;
2245  case Intrinsic::experimental_gc_statepoint:
2246  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2247  break;
2248  }
2250  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2251  // Eventually we will support lowering the @llvm.experimental.deoptimize
2252  // intrinsic, and right now there are no plans to support other intrinsics
2253  // with deopt state.
2254  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2255  } else {
2256  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2257  }
2258 
2259  // If the value of the invoke is used outside of its defining block, make it
2260  // available as a virtual register.
2261  // We already took care of the exported value for the statepoint instruction
2262  // during call to the LowerStatepoint.
2263  if (!isStatepoint(I)) {
2264  CopyToExportRegsIfNeeded(&I);
2265  }
2266 
2268  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2269  BranchProbability EHPadBBProb =
2270  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2272  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2273 
2274  // Update successor info.
2275  addSuccessorWithProb(InvokeMBB, Return);
2276  for (auto &UnwindDest : UnwindDests) {
2277  UnwindDest.first->setIsEHPad();
2278  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2279  }
2280  InvokeMBB->normalizeSuccProbs();
2281 
2282  // Drop into normal successor.
2283  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2284  MVT::Other, getControlRoot(),
2285  DAG.getBasicBlock(Return)));
2286 }
2287 
2288 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2289  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2290 }
2291 
2292 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2293  assert(FuncInfo.MBB->isEHPad() &&
2294  "Call to landingpad not in landing pad!");
2295 
2297  addLandingPadInfo(LP, *MBB);
2298 
2299  // If there aren't registers to copy the values into (e.g., during SjLj
2300  // exceptions), then don't bother to create these DAG nodes.
2301  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2302  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2303  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2304  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2305  return;
2306 
2307  // If landingpad's return type is token type, we don't create DAG nodes
2308  // for its exception pointer and selector value. The extraction of exception
2309  // pointer or selector value from token type landingpads is not currently
2310  // supported.
2311  if (LP.getType()->isTokenTy())
2312  return;
2313 
2314  SmallVector<EVT, 2> ValueVTs;
2315  SDLoc dl = getCurSDLoc();
2316  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2317  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2318 
2319  // Get the two live-in registers as SDValues. The physregs have already been
2320  // copied into virtual registers.
2321  SDValue Ops[2];
2323  Ops[0] = DAG.getZExtOrTrunc(
2326  TLI.getPointerTy(DAG.getDataLayout())),
2327  dl, ValueVTs[0]);
2328  } else {
2329  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2330  }
2331  Ops[1] = DAG.getZExtOrTrunc(
2334  TLI.getPointerTy(DAG.getDataLayout())),
2335  dl, ValueVTs[1]);
2336 
2337  // Merge into one.
2339  DAG.getVTList(ValueVTs), Ops);
2340  setValue(&LP, Res);
2341 }
2342 
2343 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2344 #ifndef NDEBUG
2345  for (const CaseCluster &CC : Clusters)
2346  assert(CC.Low == CC.High && "Input clusters must be single-case");
2347 #endif
2348 
2349  std::sort(Clusters.begin(), Clusters.end(),
2350  [](const CaseCluster &a, const CaseCluster &b) {
2351  return a.Low->getValue().slt(b.Low->getValue());
2352  });
2353 
2354  // Merge adjacent clusters with the same destination.
2355  const unsigned N = Clusters.size();
2356  unsigned DstIndex = 0;
2357  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2358  CaseCluster &CC = Clusters[SrcIndex];
2359  const ConstantInt *CaseVal = CC.Low;
2360  MachineBasicBlock *Succ = CC.MBB;
2361 
2362  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2363  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2364  // If this case has the same successor and is a neighbour, merge it into
2365  // the previous cluster.
2366  Clusters[DstIndex - 1].High = CaseVal;
2367  Clusters[DstIndex - 1].Prob += CC.Prob;
2368  } else {
2369  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2370  sizeof(Clusters[SrcIndex]));
2371  }
2372  }
2373  Clusters.resize(DstIndex);
2374 }
2375 
2377  MachineBasicBlock *Last) {
2378  // Update JTCases.
2379  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2380  if (JTCases[i].first.HeaderBB == First)
2381  JTCases[i].first.HeaderBB = Last;
2382 
2383  // Update BitTestCases.
2384  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2385  if (BitTestCases[i].Parent == First)
2386  BitTestCases[i].Parent = Last;
2387 }
2388 
2389 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2390  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2391 
2392  // Update machine-CFG edges with unique successors.
2394  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2395  BasicBlock *BB = I.getSuccessor(i);
2396  bool Inserted = Done.insert(BB).second;
2397  if (!Inserted)
2398  continue;
2399 
2400  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2401  addSuccessorWithProb(IndirectBrMBB, Succ);
2402  }
2403  IndirectBrMBB->normalizeSuccProbs();
2404 
2407  getValue(I.getAddress())));
2408 }
2409 
2410 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2412  DAG.setRoot(
2414 }
2415 
2416 void SelectionDAGBuilder::visitFSub(const User &I) {
2417  // -0.0 - X --> fneg
2418  Type *Ty = I.getType();
2419  if (isa<Constant>(I.getOperand(0)) &&
2421  SDValue Op2 = getValue(I.getOperand(1));
2423  Op2.getValueType(), Op2));
2424  return;
2425  }
2426 
2427  visitBinary(I, ISD::FSUB);
2428 }
2429 
2430 /// Checks if the given instruction performs a vector reduction, in which case
2431 /// we have the freedom to alter the elements in the result as long as the
2432 /// reduction of them stays unchanged.
2433 static bool isVectorReductionOp(const User *I) {
2434  const Instruction *Inst = dyn_cast<Instruction>(I);
2435  if (!Inst || !Inst->getType()->isVectorTy())
2436  return false;
2437 
2438  auto OpCode = Inst->getOpcode();
2439  switch (OpCode) {
2440  case Instruction::Add:
2441  case Instruction::Mul:
2442  case Instruction::And:
2443  case Instruction::Or:
2444  case Instruction::Xor:
2445  break;
2446  case Instruction::FAdd:
2447  case Instruction::FMul:
2448  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2449  if (FPOp->getFastMathFlags().unsafeAlgebra())
2450  break;
2452  default:
2453  return false;
2454  }
2455 
2456  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2457  unsigned ElemNumToReduce = ElemNum;
2458 
2459  // Do DFS search on the def-use chain from the given instruction. We only
2460  // allow four kinds of operations during the search until we reach the
2461  // instruction that extracts the first element from the vector:
2462  //
2463  // 1. The reduction operation of the same opcode as the given instruction.
2464  //
2465  // 2. PHI node.
2466  //
2467  // 3. ShuffleVector instruction together with a reduction operation that
2468  // does a partial reduction.
2469  //
2470  // 4. ExtractElement that extracts the first element from the vector, and we
2471  // stop searching the def-use chain here.
2472  //
2473  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2474  // from 1-3 to the stack to continue the DFS. The given instruction is not
2475  // a reduction operation if we meet any other instructions other than those
2476  // listed above.
2477 
2478  SmallVector<const User *, 16> UsersToVisit{Inst};
2480  bool ReduxExtracted = false;
2481 
2482  while (!UsersToVisit.empty()) {
2483  auto User = UsersToVisit.back();
2484  UsersToVisit.pop_back();
2485  if (!Visited.insert(User).second)
2486  continue;
2487 
2488  for (const auto &U : User->users()) {
2489  auto Inst = dyn_cast<Instruction>(U);
2490  if (!Inst)
2491  return false;
2492 
2493  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2494  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2495  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2496  return false;
2497  UsersToVisit.push_back(U);
2498  } else if (const ShuffleVectorInst *ShufInst =
2499  dyn_cast<ShuffleVectorInst>(U)) {
2500  // Detect the following pattern: A ShuffleVector instruction together
2501  // with a reduction that do partial reduction on the first and second
2502  // ElemNumToReduce / 2 elements, and store the result in
2503  // ElemNumToReduce / 2 elements in another vector.
2504 
2505  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2506  if (ResultElements < ElemNum)
2507  return false;
2508 
2509  if (ElemNumToReduce == 1)
2510  return false;
2511  if (!isa<UndefValue>(U->getOperand(1)))
2512  return false;
2513  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2514  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2515  return false;
2516  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2517  if (ShufInst->getMaskValue(i) != -1)
2518  return false;
2519 
2520  // There is only one user of this ShuffleVector instruction, which
2521  // must be a reduction operation.
2522  if (!U->hasOneUse())
2523  return false;
2524 
2525  auto U2 = dyn_cast<Instruction>(*U->user_begin());
2526  if (!U2 || U2->getOpcode() != OpCode)
2527  return false;
2528 
2529  // Check operands of the reduction operation.
2530  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2531  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2532  UsersToVisit.push_back(U2);
2533  ElemNumToReduce /= 2;
2534  } else
2535  return false;
2536  } else if (isa<ExtractElementInst>(U)) {
2537  // At this moment we should have reduced all elements in the vector.
2538  if (ElemNumToReduce != 1)
2539  return false;
2540 
2541  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2542  if (!Val || Val->getZExtValue() != 0)
2543  return false;
2544 
2545  ReduxExtracted = true;
2546  } else
2547  return false;
2548  }
2549  }
2550  return ReduxExtracted;
2551 }
2552 
2553 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2554  SDValue Op1 = getValue(I.getOperand(0));
2555  SDValue Op2 = getValue(I.getOperand(1));
2556 
2557  bool nuw = false;
2558  bool nsw = false;
2559  bool exact = false;
2560  bool vec_redux = false;
2561  FastMathFlags FMF;
2562 
2563  if (const OverflowingBinaryOperator *OFBinOp =
2564  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2565  nuw = OFBinOp->hasNoUnsignedWrap();
2566  nsw = OFBinOp->hasNoSignedWrap();
2567  }
2568  if (const PossiblyExactOperator *ExactOp =
2569  dyn_cast<const PossiblyExactOperator>(&I))
2570  exact = ExactOp->isExact();
2571  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2572  FMF = FPOp->getFastMathFlags();
2573 
2574  if (isVectorReductionOp(&I)) {
2575  vec_redux = true;
2576  DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2577  }
2578 
2580  Flags.setExact(exact);
2581  Flags.setNoSignedWrap(nsw);
2582  Flags.setNoUnsignedWrap(nuw);
2583  Flags.setVectorReduction(vec_redux);
2584  if (EnableFMFInDAG) {
2585  Flags.setAllowReciprocal(FMF.allowReciprocal());
2586  Flags.setNoInfs(FMF.noInfs());
2587  Flags.setNoNaNs(FMF.noNaNs());
2588  Flags.setNoSignedZeros(FMF.noSignedZeros());
2589  Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2590  }
2591  SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2592  Op1, Op2, &Flags);
2593  setValue(&I, BinNodeValue);
2594 }
2595 
2596 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2597  SDValue Op1 = getValue(I.getOperand(0));
2598  SDValue Op2 = getValue(I.getOperand(1));
2599 
2601  Op2.getValueType(), DAG.getDataLayout());
2602 
2603  // Coerce the shift amount to the right type if we can.
2604  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2605  unsigned ShiftSize = ShiftTy.getSizeInBits();
2606  unsigned Op2Size = Op2.getValueSizeInBits();
2607  SDLoc DL = getCurSDLoc();
2608 
2609  // If the operand is smaller than the shift count type, promote it.
2610  if (ShiftSize > Op2Size)
2611  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2612 
2613  // If the operand is larger than the shift count type but the shift
2614  // count type has enough bits to represent any shift value, truncate
2615  // it now. This is a common case and it exposes the truncate to
2616  // optimization early.
2617  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2618  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2619  // Otherwise we'll need to temporarily settle for some other convenient
2620  // type. Type legalization will make adjustments once the shiftee is split.
2621  else
2622  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2623  }
2624 
2625  bool nuw = false;
2626  bool nsw = false;
2627  bool exact = false;
2628 
2629  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2630 
2631  if (const OverflowingBinaryOperator *OFBinOp =
2632  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2633  nuw = OFBinOp->hasNoUnsignedWrap();
2634  nsw = OFBinOp->hasNoSignedWrap();
2635  }
2636  if (const PossiblyExactOperator *ExactOp =
2637  dyn_cast<const PossiblyExactOperator>(&I))
2638  exact = ExactOp->isExact();
2639  }
2641  Flags.setExact(exact);
2642  Flags.setNoSignedWrap(nsw);
2643  Flags.setNoUnsignedWrap(nuw);
2644  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2645  &Flags);
2646  setValue(&I, Res);
2647 }
2648 
2649 void SelectionDAGBuilder::visitSDiv(const User &I) {
2650  SDValue Op1 = getValue(I.getOperand(0));
2651  SDValue Op2 = getValue(I.getOperand(1));
2652 
2654  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2655  cast<PossiblyExactOperator>(&I)->isExact());
2657  Op2, &Flags));
2658 }
2659 
2660 void SelectionDAGBuilder::visitICmp(const User &I) {
2662  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2663  predicate = IC->getPredicate();
2664  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2665  predicate = ICmpInst::Predicate(IC->getPredicate());
2666  SDValue Op1 = getValue(I.getOperand(0));
2667  SDValue Op2 = getValue(I.getOperand(1));
2668  ISD::CondCode Opcode = getICmpCondCode(predicate);
2669 
2671  I.getType());
2672  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2673 }
2674 
2675 void SelectionDAGBuilder::visitFCmp(const User &I) {
2677  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2678  predicate = FC->getPredicate();
2679  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2680  predicate = FCmpInst::Predicate(FC->getPredicate());
2681  SDValue Op1 = getValue(I.getOperand(0));
2682  SDValue Op2 = getValue(I.getOperand(1));
2683  ISD::CondCode Condition = getFCmpCondCode(predicate);
2684 
2685  // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2686  // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2687  // further optimization, but currently FMF is only applicable to binary nodes.
2688  if (TM.Options.NoNaNsFPMath)
2689  Condition = getFCmpCodeWithoutNaN(Condition);
2691  I.getType());
2692  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2693 }
2694 
2695 // Check if the condition of the select has one use or two users that are both
2696 // selects with the same condition.
2697 static bool hasOnlySelectUsers(const Value *Cond) {
2698  return all_of(Cond->users(), [](const Value *V) {
2699  return isa<SelectInst>(V);
2700  });
2701 }
2702 
2703 void SelectionDAGBuilder::visitSelect(const User &I) {
2704  SmallVector<EVT, 4> ValueVTs;
2706  ValueVTs);
2707  unsigned NumValues = ValueVTs.size();
2708  if (NumValues == 0) return;
2709 
2710  SmallVector<SDValue, 4> Values(NumValues);
2711  SDValue Cond = getValue(I.getOperand(0));
2712  SDValue LHSVal = getValue(I.getOperand(1));
2713  SDValue RHSVal = getValue(I.getOperand(2));
2714  auto BaseOps = {Cond};
2715  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2717 
2718  // Min/max matching is only viable if all output VTs are the same.
2719  if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2720  EVT VT = ValueVTs[0];
2721  LLVMContext &Ctx = *DAG.getContext();
2722  auto &TLI = DAG.getTargetLoweringInfo();
2723 
2724  // We care about the legality of the operation after it has been type
2725  // legalized.
2726  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2727  VT != TLI.getTypeToTransformTo(Ctx, VT))
2728  VT = TLI.getTypeToTransformTo(Ctx, VT);
2729 
2730  // If the vselect is legal, assume we want to leave this as a vector setcc +
2731  // vselect. Otherwise, if this is going to be scalarized, we want to see if
2732  // min/max is legal on the scalar type.
2733  bool UseScalarMinMax = VT.isVector() &&
2735 
2736  Value *LHS, *RHS;
2737  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2739  switch (SPR.Flavor) {
2740  case SPF_UMAX: Opc = ISD::UMAX; break;
2741  case SPF_UMIN: Opc = ISD::UMIN; break;
2742  case SPF_SMAX: Opc = ISD::SMAX; break;
2743  case SPF_SMIN: Opc = ISD::SMIN; break;
2744  case SPF_FMINNUM:
2745  switch (SPR.NaNBehavior) {
2746  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2747  case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2748  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2749  case SPNB_RETURNS_ANY: {
2751  Opc = ISD::FMINNUM;
2752  else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2753  Opc = ISD::FMINNAN;
2754  else if (UseScalarMinMax)
2757  break;
2758  }
2759  }
2760  break;
2761  case SPF_FMAXNUM:
2762  switch (SPR.NaNBehavior) {
2763  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2764  case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2765  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2766  case SPNB_RETURNS_ANY:
2767 
2769  Opc = ISD::FMAXNUM;
2770  else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2771  Opc = ISD::FMAXNAN;
2772  else if (UseScalarMinMax)
2775  break;
2776  }
2777  break;
2778  default: break;
2779  }
2780 
2781  if (Opc != ISD::DELETED_NODE &&
2782  (TLI.isOperationLegalOrCustom(Opc, VT) ||
2783  (UseScalarMinMax &&
2784  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2785  // If the underlying comparison instruction is used by any other
2786  // instruction, the consumed instructions won't be destroyed, so it is
2787  // not profitable to convert to a min/max.
2788  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2789  OpCode = Opc;
2790  LHSVal = getValue(LHS);
2791  RHSVal = getValue(RHS);
2792  BaseOps = {};
2793  }
2794  }
2795 
2796  for (unsigned i = 0; i != NumValues; ++i) {
2797  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2798  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2799  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2800  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2801  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2802  Ops);
2803  }
2804 
2806  DAG.getVTList(ValueVTs), Values));
2807 }
2808 
2809 void SelectionDAGBuilder::visitTrunc(const User &I) {
2810  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2811  SDValue N = getValue(I.getOperand(0));
2813  I.getType());
2814  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2815 }
2816 
2817 void SelectionDAGBuilder::visitZExt(const User &I) {
2818  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2819  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2820  SDValue N = getValue(I.getOperand(0));
2822  I.getType());
2823  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2824 }
2825 
2826 void SelectionDAGBuilder::visitSExt(const User &I) {
2827  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2828  // SExt also can't be a cast to bool for same reason. So, nothing much to do
2829  SDValue N = getValue(I.getOperand(0));
2831  I.getType());
2832  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2833 }
2834 
2835 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2836  // FPTrunc is never a no-op cast, no need to check
2837  SDValue N = getValue(I.getOperand(0));
2838  SDLoc dl = getCurSDLoc();
2839  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2840  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2841  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2843  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2844 }
2845 
2846 void SelectionDAGBuilder::visitFPExt(const User &I) {
2847  // FPExt is never a no-op cast, no need to check
2848  SDValue N = getValue(I.getOperand(0));
2850  I.getType());
2851  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2852 }
2853 
2854 void SelectionDAGBuilder::visitFPToUI(const User &I) {
2855  // FPToUI is never a no-op cast, no need to check
2856  SDValue N = getValue(I.getOperand(0));
2858  I.getType());
2859  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2860 }
2861 
2862 void SelectionDAGBuilder::visitFPToSI(const User &I) {
2863  // FPToSI is never a no-op cast, no need to check
2864  SDValue N = getValue(I.getOperand(0));
2866  I.getType());
2867  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2868 }
2869 
2870 void SelectionDAGBuilder::visitUIToFP(const User &I) {
2871  // UIToFP is never a no-op cast, no need to check
2872  SDValue N = getValue(I.getOperand(0));
2874  I.getType());
2875  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2876 }
2877 
2878 void SelectionDAGBuilder::visitSIToFP(const User &I) {
2879  // SIToFP is never a no-op cast, no need to check
2880  SDValue N = getValue(I.getOperand(0));
2882  I.getType());
2883  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2884 }
2885 
2886 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2887  // What to do depends on the size of the integer and the size of the pointer.
2888  // We can either truncate, zero extend, or no-op, accordingly.
2889  SDValue N = getValue(I.getOperand(0));
2891  I.getType());
2892  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2893 }
2894 
2895 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2896  // What to do depends on the size of the integer and the size of the pointer.
2897  // We can either truncate, zero extend, or no-op, accordingly.
2898  SDValue N = getValue(I.getOperand(0));
2900  I.getType());
2901  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2902 }
2903 
2904 void SelectionDAGBuilder::visitBitCast(const User &I) {
2905  SDValue N = getValue(I.getOperand(0));
2906  SDLoc dl = getCurSDLoc();
2908  I.getType());
2909 
2910  // BitCast assures us that source and destination are the same size so this is
2911  // either a BITCAST or a no-op.
2912  if (DestVT != N.getValueType())
2913  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2914  DestVT, N)); // convert types.
2915  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2916  // might fold any kind of constant expression to an integer constant and that
2917  // is not what we are looking for. Only regcognize a bitcast of a genuine
2918  // constant integer as an opaque constant.
2919  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2920  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2921  /*isOpaque*/true));
2922  else
2923  setValue(&I, N); // noop cast.
2924 }
2925 
2926 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2927  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2928  const Value *SV = I.getOperand(0);
2929  SDValue N = getValue(SV);
2930  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2931 
2932  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
2933  unsigned DestAS = I.getType()->getPointerAddressSpace();
2934 
2935  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
2936  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
2937 
2938  setValue(&I, N);
2939 }
2940 
2941 void SelectionDAGBuilder::visitInsertElement(const User &I) {
2942  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2943  SDValue InVec = getValue(I.getOperand(0));
2944  SDValue InVal = getValue(I.getOperand(1));
2948  TLI.getValueType(DAG.getDataLayout(), I.getType()),
2949  InVec, InVal, InIdx));
2950 }
2951 
2952 void SelectionDAGBuilder::visitExtractElement(const User &I) {
2953  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2954  SDValue InVec = getValue(I.getOperand(0));
2958  TLI.getValueType(DAG.getDataLayout(), I.getType()),
2959  InVec, InIdx));
2960 }
2961 
2962 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
2963  SDValue Src1 = getValue(I.getOperand(0));
2964  SDValue Src2 = getValue(I.getOperand(1));
2965  SDLoc DL = getCurSDLoc();
2966 
2968  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
2969  unsigned MaskNumElts = Mask.size();
2970 
2971  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2972  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2973  EVT SrcVT = Src1.getValueType();
2974  unsigned SrcNumElts = SrcVT.getVectorNumElements();
2975 
2976  if (SrcNumElts == MaskNumElts) {
2977  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
2978  return;
2979  }
2980 
2981  // Normalize the shuffle vector since mask and vector length don't match.
2982  if (SrcNumElts < MaskNumElts) {
2983  // Mask is longer than the source vectors. We can use concatenate vector to
2984  // make the mask and vectors lengths match.
2985 
2986  if (MaskNumElts % SrcNumElts == 0) {
2987  // Mask length is a multiple of the source vector length.
2988  // Check if the shuffle is some kind of concatenation of the input
2989  // vectors.
2990  unsigned NumConcat = MaskNumElts / SrcNumElts;
2991  bool IsConcat = true;
2992  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
2993  for (unsigned i = 0; i != MaskNumElts; ++i) {
2994  int Idx = Mask[i];
2995  if (Idx < 0)
2996  continue;
2997  // Ensure the indices in each SrcVT sized piece are sequential and that
2998  // the same source is used for the whole piece.
2999  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3000  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3001  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3002  IsConcat = false;
3003  break;
3004  }
3005  // Remember which source this index came from.
3006  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3007  }
3008 
3009  // The shuffle is concatenating multiple vectors together. Just emit
3010  // a CONCAT_VECTORS operation.
3011  if (IsConcat) {
3012  SmallVector<SDValue, 8> ConcatOps;
3013  for (auto Src : ConcatSrcs) {
3014  if (Src < 0)
3015  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3016  else if (Src == 0)
3017  ConcatOps.push_back(Src1);
3018  else
3019  ConcatOps.push_back(Src2);
3020  }
3021  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3022  return;
3023  }
3024  }
3025 
3026  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3027  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3028  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3029  PaddedMaskNumElts);
3030 
3031  // Pad both vectors with undefs to make them the same length as the mask.
3032  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3033 
3034  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3035  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3036  MOps1[0] = Src1;
3037  MOps2[0] = Src2;
3038 
3039  Src1 = Src1.isUndef()
3040  ? DAG.getUNDEF(PaddedVT)
3041  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3042  Src2 = Src2.isUndef()
3043  ? DAG.getUNDEF(PaddedVT)
3044  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3045 
3046  // Readjust mask for new input vector length.
3047  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3048  for (unsigned i = 0; i != MaskNumElts; ++i) {
3049  int Idx = Mask[i];
3050  if (Idx >= (int)SrcNumElts)
3051  Idx -= SrcNumElts - PaddedMaskNumElts;
3052  MappedOps[i] = Idx;
3053  }
3054 
3055  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3056 
3057  // If the concatenated vector was padded, extract a subvector with the
3058  // correct number of elements.
3059  if (MaskNumElts != PaddedMaskNumElts)
3060  Result = DAG.getNode(
3061  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3063 
3064  setValue(&I, Result);
3065  return;
3066  }
3067 
3068  if (SrcNumElts > MaskNumElts) {
3069  // Analyze the access pattern of the vector to see if we can extract
3070  // two subvectors and do the shuffle. The analysis is done by calculating
3071  // the range of elements the mask access on both vectors.
3072  int MinRange[2] = { static_cast<int>(SrcNumElts),
3073  static_cast<int>(SrcNumElts)};
3074  int MaxRange[2] = {-1, -1};
3075 
3076  for (unsigned i = 0; i != MaskNumElts; ++i) {
3077  int Idx = Mask[i];
3078  unsigned Input = 0;
3079  if (Idx < 0)
3080  continue;
3081 
3082  if (Idx >= (int)SrcNumElts) {
3083  Input = 1;
3084  Idx -= SrcNumElts;
3085  }
3086  if (Idx > MaxRange[Input])
3087  MaxRange[Input] = Idx;
3088  if (Idx < MinRange[Input])
3089  MinRange[Input] = Idx;
3090  }
3091 
3092  // Check if the access is smaller than the vector size and can we find
3093  // a reasonable extract index.
3094  int RangeUse[2] = { -1, -1 }; // 0 = Unused, 1 = Extract, -1 = Can not
3095  // Extract.
3096  int StartIdx[2]; // StartIdx to extract from
3097  for (unsigned Input = 0; Input < 2; ++Input) {
3098  if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
3099  RangeUse[Input] = 0; // Unused
3100  StartIdx[Input] = 0;
3101  continue;
3102  }
3103 
3104  // Find a good start index that is a multiple of the mask length. Then
3105  // see if the rest of the elements are in range.
3106  StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
3107  if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
3108  StartIdx[Input] + MaskNumElts <= SrcNumElts)
3109  RangeUse[Input] = 1; // Extract from a multiple of the mask length.
3110  }
3111 
3112  if (RangeUse[0] == 0 && RangeUse[1] == 0) {
3113  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3114  return;
3115  }
3116  if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
3117  // Extract appropriate subvector and generate a vector shuffle
3118  for (unsigned Input = 0; Input < 2; ++Input) {
3119  SDValue &Src = Input == 0 ? Src1 : Src2;
3120  if (RangeUse[Input] == 0)
3121  Src = DAG.getUNDEF(VT);
3122  else {
3123  Src = DAG.getNode(
3124  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3125  DAG.getConstant(StartIdx[Input], DL,
3126  TLI.getVectorIdxTy(DAG.getDataLayout())));
3127  }
3128  }
3129 
3130  // Calculate new mask.
3131  SmallVector<int, 8> MappedOps;
3132  for (unsigned i = 0; i != MaskNumElts; ++i) {
3133  int Idx = Mask[i];
3134  if (Idx >= 0) {
3135  if (Idx < (int)SrcNumElts)
3136  Idx -= StartIdx[0];
3137  else
3138  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3139  }
3140  MappedOps.push_back(Idx);
3141  }
3142 
3143  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3144  return;
3145  }
3146  }
3147 
3148  // We can't use either concat vectors or extract subvectors so fall back to
3149  // replacing the shuffle with extract and build vector.
3150  // to insert and build vector.
3151  EVT EltVT = VT.getVectorElementType();
3152  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3154  for (unsigned i = 0; i != MaskNumElts; ++i) {
3155  int Idx = Mask[i];
3156  SDValue Res;
3157 
3158  if (Idx < 0) {
3159  Res = DAG.getUNDEF(EltVT);
3160  } else {
3161  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3162  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3163 
3165  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3166  }
3167 
3168  Ops.push_back(Res);
3169  }
3170 
3171  setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops));
3172 }
3173 
3174 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3175  const Value *Op0 = I.getOperand(0);
3176  const Value *Op1 = I.getOperand(1);
3177  Type *AggTy = I.getType();
3178  Type *ValTy = Op1->getType();
3179  bool IntoUndef = isa<UndefValue>(Op0);
3180  bool FromUndef = isa<UndefValue>(Op1);
3181 
3182  unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3183 
3184  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3185  SmallVector<EVT, 4> AggValueVTs;
3186  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3187  SmallVector<EVT, 4> ValValueVTs;
3188  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3189 
3190  unsigned NumAggValues = AggValueVTs.size();
3191  unsigned NumValValues = ValValueVTs.size();
3192  SmallVector<SDValue, 4> Values(NumAggValues);
3193 
3194  // Ignore an insertvalue that produces an empty object
3195  if (!NumAggValues) {
3197  return;
3198  }
3199 
3200  SDValue Agg = getValue(Op0);
3201  unsigned i = 0;
3202  // Copy the beginning value(s) from the original aggregate.
3203  for (; i != LinearIndex; ++i)
3204  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3205  SDValue(Agg.getNode(), Agg.getResNo() + i);
3206  // Copy values from the inserted value(s).
3207  if (NumValValues) {
3208  SDValue Val = getValue(Op1);
3209  for (; i != LinearIndex + NumValValues; ++i)
3210  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3211  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3212  }
3213  // Copy remaining value(s) from the original aggregate.
3214  for (; i != NumAggValues; ++i)
3215  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3216  SDValue(Agg.getNode(), Agg.getResNo() + i);
3217 
3219  DAG.getVTList(AggValueVTs), Values));
3220 }
3221 
3222 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3223  const Value *Op0 = I.getOperand(0);
3224  Type *AggTy = Op0->getType();
3225  Type *ValTy = I.getType();
3226  bool OutOfUndef = isa<UndefValue>(Op0);
3227 
3228  unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3229 
3230  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3231  SmallVector<EVT, 4> ValValueVTs;
3232  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3233 
3234  unsigned NumValValues = ValValueVTs.size();
3235 
3236  // Ignore a extractvalue that produces an empty object
3237  if (!NumValValues) {
3239  return;
3240  }
3241 
3242  SmallVector<SDValue, 4> Values(NumValValues);
3243 
3244  SDValue Agg = getValue(Op0);
3245  // Copy out the selected value(s).
3246  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3247  Values[i - LinearIndex] =
3248  OutOfUndef ?
3249  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3250  SDValue(Agg.getNode(), Agg.getResNo() + i);
3251 
3253  DAG.getVTList(ValValueVTs), Values));
3254 }
3255 
3256 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3257  Value *Op0 = I.getOperand(0);
3258  // Note that the pointer operand may be a vector of pointers. Take the scalar
3259  // element which holds a pointer.
3260  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3261  SDValue N = getValue(Op0);
3262  SDLoc dl = getCurSDLoc();
3263 
3264  // Normalize Vector GEP - all scalar operands should be converted to the
3265  // splat vector.
3266  unsigned VectorWidth = I.getType()->isVectorTy() ?
3267  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3268 
3269  if (VectorWidth && !N.getValueType().isVector()) {
3270  LLVMContext &Context = *DAG.getContext();
3271  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3272  N = DAG.getSplatBuildVector(VT, dl, N);
3273  }
3274 
3275  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3276  GTI != E; ++GTI) {
3277  const Value *Idx = GTI.getOperand();
3278  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3279  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3280  if (Field) {
3281  // N = N + Offset
3282  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3283 
3284  // In an inbouds GEP with an offset that is nonnegative even when
3285  // interpreted as signed, assume there is no unsigned overflow.
3287  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3288  Flags.setNoUnsignedWrap(true);
3289 
3290  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3291  DAG.getConstant(Offset, dl, N.getValueType()), &Flags);
3292  }
3293  } else {
3294  MVT PtrTy =
3296  unsigned PtrSize = PtrTy.getSizeInBits();
3297  APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3298 
3299  // If this is a scalar constant or a splat vector of constants,
3300  // handle it quickly.
3301  const auto *CI = dyn_cast<ConstantInt>(Idx);
3302  if (!CI && isa<ConstantDataVector>(Idx) &&
3303  cast<ConstantDataVector>(Idx)->getSplatValue())
3304  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3305 
3306  if (CI) {
3307  if (CI->isZero())
3308  continue;
3309  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3310  LLVMContext &Context = *DAG.getContext();
3311  SDValue OffsVal = VectorWidth ?
3312  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3313  DAG.getConstant(Offs, dl, PtrTy);
3314 
3315  // In an inbouds GEP with an offset that is nonnegative even when
3316  // interpreted as signed, assume there is no unsigned overflow.
3318  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3319  Flags.setNoUnsignedWrap(true);
3320 
3321  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, &Flags);
3322  continue;
3323  }
3324 
3325  // N = N + Idx * ElementSize;
3326  SDValue IdxN = getValue(Idx);
3327 
3328  if (!IdxN.getValueType().isVector() && VectorWidth) {
3329  MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
3330  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3331  }
3332 
3333  // If the index is smaller or larger than intptr_t, truncate or extend
3334  // it.
3335  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3336 
3337  // If this is a multiply by a power of two, turn it into a shl
3338  // immediately. This is a very common case.
3339  if (ElementSize != 1) {
3340  if (ElementSize.isPowerOf2()) {
3341  unsigned Amt = ElementSize.logBase2();
3342  IdxN = DAG.getNode(ISD::SHL, dl,
3343  N.getValueType(), IdxN,
3344  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3345  } else {
3346  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3347  IdxN = DAG.getNode(ISD::MUL, dl,
3348  N.getValueType(), IdxN, Scale);
3349  }
3350  }
3351 
3352  N = DAG.getNode(ISD::ADD, dl,
3353  N.getValueType(), N, IdxN);
3354  }
3355  }
3356 
3357  setValue(&I, N);
3358 }
3359 
3360 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3361  // If this is a fixed sized alloca in the entry block of the function,
3362  // allocate it statically on the stack.
3363  if (FuncInfo.StaticAllocaMap.count(&I))
3364  return; // getValue will auto-populate this.
3365 
3366  SDLoc dl = getCurSDLoc();
3367  Type *Ty = I.getAllocatedType();
3368  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3369  auto &DL = DAG.getDataLayout();
3370  uint64_t TySize = DL.getTypeAllocSize(Ty);
3371  unsigned Align =
3372  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3373 
3374  SDValue AllocSize = getValue(I.getArraySize());
3375 
3376  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3377  if (AllocSize.getValueType() != IntPtr)
3378  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3379 
3380  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3381  AllocSize,
3382  DAG.getConstant(TySize, dl, IntPtr));
3383 
3384  // Handle alignment. If the requested alignment is less than or equal to
3385  // the stack alignment, ignore it. If the size is greater than or equal to
3386  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3387  unsigned StackAlign =
3389  if (Align <= StackAlign)
3390  Align = 0;
3391 
3392  // Round the size of the allocation up to the stack alignment size
3393  // by add SA-1 to the size. This doesn't overflow because we're computing
3394  // an address inside an alloca.
3396  Flags.setNoUnsignedWrap(true);
3397  AllocSize = DAG.getNode(ISD::ADD, dl,
3398  AllocSize.getValueType(), AllocSize,
3399  DAG.getIntPtrConstant(StackAlign - 1, dl), &Flags);
3400 
3401  // Mask out the low bits for alignment purposes.
3402  AllocSize = DAG.getNode(ISD::AND, dl,
3403  AllocSize.getValueType(), AllocSize,
3404  DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3405  dl));
3406 
3407  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3408  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3409  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3410  setValue(&I, DSA);
3411  DAG.setRoot(DSA.getValue(1));
3412 
3414 }
3415 
3416 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3417  if (I.isAtomic())
3418  return visitAtomicLoad(I);
3419 
3420  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3421  const Value *SV = I.getOperand(0);
3422  if (TLI.supportSwiftError()) {
3423  // Swifterror values can come from either a function parameter with
3424  // swifterror attribute or an alloca with swifterror attribute.
3425  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3426  if (Arg->hasSwiftErrorAttr())
3427  return visitLoadFromSwiftError(I);
3428  }
3429 
3430  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3431  if (Alloca->isSwiftError())
3432  return visitLoadFromSwiftError(I);
3433  }
3434  }
3435 
3436  SDValue Ptr = getValue(SV);
3437 
3438  Type *Ty = I.getType();
3439 
3440  bool isVolatile = I.isVolatile();
3441  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3442  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3443  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3444  unsigned Alignment = I.getAlignment();
3445 
3446  AAMDNodes AAInfo;
3447  I.getAAMetadata(AAInfo);
3448  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3449 
3450  SmallVector<EVT, 4> ValueVTs;
3452  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3453  unsigned NumValues = ValueVTs.size();
3454  if (NumValues == 0)
3455  return;
3456 
3457  SDValue Root;
3458  bool ConstantMemory = false;
3459  if (isVolatile || NumValues > MaxParallelChains)
3460  // Serialize volatile loads with other side effects.
3461  Root = getRoot();
3463  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3464  // Do not serialize (non-volatile) loads of constant memory with anything.
3465  Root = DAG.getEntryNode();
3466  ConstantMemory = true;
3467  } else {
3468  // Do not serialize non-volatile loads against each other.
3469  Root = DAG.getRoot();
3470  }
3471 
3472  SDLoc dl = getCurSDLoc();
3473 
3474  if (isVolatile)
3475  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3476 
3477  // An aggregate load cannot wrap around the address space, so offsets to its
3478  // parts don't wrap either.
3480  Flags.setNoUnsignedWrap(true);
3481 
3482  SmallVector<SDValue, 4> Values(NumValues);
3484  EVT PtrVT = Ptr.getValueType();
3485  unsigned ChainI = 0;
3486  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3487  // Serializing loads here may result in excessive register pressure, and
3488  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3489  // could recover a bit by hoisting nodes upward in the chain by recognizing
3490  // they are side-effect free or do not alias. The optimizer should really
3491  // avoid this case by converting large object/array copies to llvm.memcpy
3492  // (MaxParallelChains should always remain as failsafe).
3493  if (ChainI == MaxParallelChains) {
3494  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3496  makeArrayRef(Chains.data(), ChainI));
3497  Root = Chain;
3498  ChainI = 0;
3499  }
3500  SDValue A = DAG.getNode(ISD::ADD, dl,
3501  PtrVT, Ptr,
3502  DAG.getConstant(Offsets[i], dl, PtrVT),
3503  &Flags);
3504  auto MMOFlags = MachineMemOperand::MONone;
3505  if (isVolatile)
3506  MMOFlags |= MachineMemOperand::MOVolatile;
3507  if (isNonTemporal)
3509  if (isInvariant)
3510  MMOFlags |= MachineMemOperand::MOInvariant;
3511  if (isDereferenceable)
3513 
3514  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3515  MachinePointerInfo(SV, Offsets[i]), Alignment,
3516  MMOFlags, AAInfo, Ranges);
3517 
3518  Values[i] = L;
3519  Chains[ChainI] = L.getValue(1);
3520  }
3521 
3522  if (!ConstantMemory) {
3524  makeArrayRef(Chains.data(), ChainI));
3525  if (isVolatile)
3526  DAG.setRoot(Chain);
3527  else
3528  PendingLoads.push_back(Chain);
3529  }
3530 
3532  DAG.getVTList(ValueVTs), Values));
3533 }
3534 
3535 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3536  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3537  assert(TLI.supportSwiftError() &&
3538  "call visitStoreToSwiftError when backend supports swifterror");
3539 
3540  SmallVector<EVT, 4> ValueVTs;
3542  const Value *SrcV = I.getOperand(0);
3544  SrcV->getType(), ValueVTs, &Offsets);
3545  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3546  "expect a single EVT for swifterror");
3547 
3548  SDValue Src = getValue(SrcV);
3549  // Create a virtual register, then update the virtual register.
3550  auto &DL = DAG.getDataLayout();
3551  const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3552  unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
3553  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3554  // Chain can be getRoot or getControlRoot.
3555  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3556  SDValue(Src.getNode(), Src.getResNo()));
3557  DAG.setRoot(CopyNode);
3559 }
3560 
3561 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3563  "call visitLoadFromSwiftError when backend supports swifterror");
3564 
3565  assert(!I.isVolatile() &&
3566  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3568  "Support volatile, non temporal, invariant for load_from_swift_error");
3569 
3570  const Value *SV = I.getOperand(0);
3571  Type *Ty = I.getType();
3572  AAMDNodes AAInfo;
3573  I.getAAMetadata(AAInfo);
3575  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) &&
3576  "load_from_swift_error should not be constant memory");
3577 
3578  SmallVector<EVT, 4> ValueVTs;
3581  ValueVTs, &Offsets);
3582  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3583  "expect a single EVT for swifterror");
3584 
3585  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3587  getRoot(), getCurSDLoc(),
3588  FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, SV), ValueVTs[0]);
3589 
3590  setValue(&I, L);
3591 }
3592 
3593 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3594  if (I.isAtomic())
3595  return visitAtomicStore(I);
3596 
3597  const Value *SrcV = I.getOperand(0);
3598  const Value *PtrV = I.getOperand(1);
3599 
3600  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3601  if (TLI.supportSwiftError()) {
3602  // Swifterror values can come from either a function parameter with
3603  // swifterror attribute or an alloca with swifterror attribute.
3604  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3605  if (Arg->hasSwiftErrorAttr())
3606  return visitStoreToSwiftError(I);
3607  }
3608 
3609  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3610  if (Alloca->isSwiftError())
3611  return visitStoreToSwiftError(I);
3612  }
3613  }
3614 
3615  SmallVector<EVT, 4> ValueVTs;
3618  SrcV->getType(), ValueVTs, &Offsets);
3619  unsigned NumValues = ValueVTs.size();
3620  if (NumValues == 0)
3621  return;
3622 
3623  // Get the lowered operands. Note that we do this after
3624  // checking if NumResults is zero, because with zero results
3625  // the operands won't have values in the map.
3626  SDValue Src = getValue(SrcV);
3627  SDValue Ptr = getValue(PtrV);
3628 
3629  SDValue Root = getRoot();
3631  SDLoc dl = getCurSDLoc();
3632  EVT PtrVT = Ptr.getValueType();
3633  unsigned Alignment = I.getAlignment();
3634  AAMDNodes AAInfo;
3635  I.getAAMetadata(AAInfo);
3636 
3637  auto MMOFlags = MachineMemOperand::MONone;
3638  if (I.isVolatile())
3639  MMOFlags |= MachineMemOperand::MOVolatile;
3640  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3642 
3643  // An aggregate load cannot wrap around the address space, so offsets to its
3644  // parts don't wrap either.
3646  Flags.setNoUnsignedWrap(true);
3647 
3648  unsigned ChainI = 0;
3649  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3650  // See visitLoad comments.
3651  if (ChainI == MaxParallelChains) {
3653  makeArrayRef(Chains.data(), ChainI));
3654  Root = Chain;
3655  ChainI = 0;
3656  }
3657  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3658  DAG.getConstant(Offsets[i], dl, PtrVT), &Flags);
3659  SDValue St = DAG.getStore(
3660  Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3661  MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3662  Chains[ChainI] = St;
3663  }
3664 
3665  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3666  makeArrayRef(Chains.data(), ChainI));
3667  DAG.setRoot(StoreNode);
3668 }
3669 
3670 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3671  bool IsCompressing) {
3672  SDLoc sdl = getCurSDLoc();
3673 
3674  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3675  unsigned& Alignment) {
3676  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3677  Src0 = I.getArgOperand(0);
3678  Ptr = I.getArgOperand(1);
3679  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3680  Mask = I.getArgOperand(3);
3681  };
3682  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3683  unsigned& Alignment) {
3684  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3685  Src0 = I.getArgOperand(0);
3686  Ptr = I.getArgOperand(1);
3687  Mask = I.getArgOperand(2);
3688  Alignment = 0;
3689  };
3690 
3691  Value *PtrOperand, *MaskOperand, *Src0Operand;
3692  unsigned Alignment;
3693  if (IsCompressing)
3694  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3695  else
3696  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3697 
3698  SDValue Ptr = getValue(PtrOperand);
3699  SDValue Src0 = getValue(Src0Operand);
3700  SDValue Mask = getValue(MaskOperand);
3701 
3702  EVT VT = Src0.getValueType();
3703  if (!Alignment)
3704  Alignment = DAG.getEVTAlignment(VT);
3705 
3706  AAMDNodes AAInfo;
3707  I.getAAMetadata(AAInfo);
3708 
3709  MachineMemOperand *MMO =
3711  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3713  Alignment, AAInfo);
3714  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3715  MMO, false /* Truncating */,
3716  IsCompressing);
3717  DAG.setRoot(StoreNode);
3718  setValue(&I, StoreNode);
3719 }
3720 
3721 // Get a uniform base for the Gather/Scatter intrinsic.
3722 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3723 // We try to represent it as a base pointer + vector of indices.
3724 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3725 // The first operand of the GEP may be a single pointer or a vector of pointers
3726 // Example:
3727 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3728 // or
3729 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3730 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3731 //
3732 // When the first GEP operand is a single pointer - it is the uniform base we
3733 // are looking for. If first operand of the GEP is a splat vector - we
3734 // extract the spalt value and use it as a uniform base.
3735 // In all other cases the function returns 'false'.
3736 //
3737 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3738  SelectionDAGBuilder* SDB) {
3739 
3740  SelectionDAG& DAG = SDB->DAG;
3741  LLVMContext &Context = *DAG.getContext();
3742 
3743  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3745  if (!GEP || GEP->getNumOperands() > 2)
3746  return false;
3747 
3748  const Value *GEPPtr = GEP->getPointerOperand();
3749  if (!GEPPtr->getType()->isVectorTy())
3750  Ptr = GEPPtr;
3751  else if (!(Ptr = getSplatValue(GEPPtr)))
3752  return false;
3753 
3754  Value *IndexVal = GEP->getOperand(1);
3755 
3756  // The operands of the GEP may be defined in another basic block.
3757  // In this case we'll not find nodes for the operands.
3758  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3759  return false;
3760 
3761  Base = SDB->getValue(Ptr);
3762  Index = SDB->getValue(IndexVal);
3763 
3764  // Suppress sign extension.
3765  if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3766  if (SDB->findValue(Sext->getOperand(0))) {
3767  IndexVal = Sext->getOperand(0);
3768  Index = SDB->getValue(IndexVal);
3769  }
3770  }
3771  if (!Index.getValueType().isVector()) {
3772  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3773  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3774  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3775  }
3776  return true;
3777 }
3778 
3779 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3780  SDLoc sdl = getCurSDLoc();
3781 
3782  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3783  const Value *Ptr = I.getArgOperand(1);
3784  SDValue Src0 = getValue(I.getArgOperand(0));
3785  SDValue Mask = getValue(I.getArgOperand(3));
3786  EVT VT = Src0.getValueType();
3787  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3788  if (!Alignment)
3789  Alignment = DAG.getEVTAlignment(VT);
3790  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3791 
3792  AAMDNodes AAInfo;
3793  I.getAAMetadata(AAInfo);
3794 
3795  SDValue Base;
3796  SDValue Index;
3797  const Value *BasePtr = Ptr;
3798  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3799 
3800  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3802  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3803  MachineMemOperand::MOStore, VT.getStoreSize(),
3804  Alignment, AAInfo);
3805  if (!UniformBase) {
3806  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3807  Index = getValue(Ptr);
3808  }
3809  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3810  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3811  Ops, MMO);
3812  DAG.setRoot(Scatter);
3813  setValue(&I, Scatter);
3814 }
3815 
3816 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3817  SDLoc sdl = getCurSDLoc();
3818 
3819  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3820  unsigned& Alignment) {
3821  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3822  Ptr = I.getArgOperand(0);
3823  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3824  Mask = I.getArgOperand(2);
3825  Src0 = I.getArgOperand(3);
3826  };
3827  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3828  unsigned& Alignment) {
3829  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3830  Ptr = I.getArgOperand(0);
3831  Alignment = 0;
3832  Mask = I.getArgOperand(1);
3833  Src0 = I.getArgOperand(2);
3834  };
3835 
3836  Value *PtrOperand, *MaskOperand, *Src0Operand;
3837  unsigned Alignment;
3838  if (IsExpanding)
3839  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3840  else
3841  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3842 
3843  SDValue Ptr = getValue(PtrOperand);
3844  SDValue Src0 = getValue(Src0Operand);
3845  SDValue Mask = getValue(MaskOperand);
3846 
3847  EVT VT = Src0.getValueType();
3848  if (!Alignment)
3849  Alignment = DAG.getEVTAlignment(VT);
3850 
3851  AAMDNodes AAInfo;
3852  I.getAAMetadata(AAInfo);
3853  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3854 
3855  // Do not serialize masked loads of constant memory with anything.
3856  bool AddToChain = !AA->pointsToConstantMemory(MemoryLocation(
3857  PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3858  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3859 
3860  MachineMemOperand *MMO =
3862  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3864  Alignment, AAInfo, Ranges);
3865 
3866  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3867  ISD::NON_EXTLOAD, IsExpanding);
3868  if (AddToChain) {
3869  SDValue OutChain = Load.getValue(1);
3870  DAG.setRoot(OutChain);
3871  }
3872  setValue(&I, Load);
3873 }
3874 
3875 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3876  SDLoc sdl = getCurSDLoc();
3877 
3878  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3879  const Value *Ptr = I.getArgOperand(0);
3880  SDValue Src0 = getValue(I.getArgOperand(3));
3881  SDValue Mask = getValue(I.getArgOperand(2));
3882 
3883  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3884  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3885  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3886  if (!Alignment)
3887  Alignment = DAG.getEVTAlignment(VT);
3888 
3889  AAMDNodes AAInfo;
3890  I.getAAMetadata(AAInfo);
3891  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3892 
3893  SDValue Root = DAG.getRoot();
3894  SDValue Base;
3895  SDValue Index;
3896  const Value *BasePtr = Ptr;
3897  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3898  bool ConstantMemory = false;
3899  if (UniformBase &&
3901  BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3902  AAInfo))) {
3903  // Do not serialize (non-volatile) loads of constant memory with anything.
3904  Root = DAG.getEntryNode();
3905  ConstantMemory = true;
3906  }
3907 
3908  MachineMemOperand *MMO =
3910  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3912  Alignment, AAInfo, Ranges);
3913 
3914  if (!UniformBase) {
3915  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3916  Index = getValue(Ptr);
3917  }
3918  SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3919  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3920  Ops, MMO);
3921 
3922  SDValue OutChain = Gather.getValue(1);
3923  if (!ConstantMemory)
3924  PendingLoads.push_back(OutChain);
3925  setValue(&I, Gather);
3926 }
3927 
3928 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3929  SDLoc dl = getCurSDLoc();
3930  AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3931  AtomicOrdering FailureOrder = I.getFailureOrdering();
3932  SynchronizationScope Scope = I.getSynchScope();
3933 
3934  SDValue InChain = getRoot();
3935 
3936  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3937  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3939  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3942  /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
3943 
3944  SDValue OutChain = L.getValue(2);
3945 
3946  setValue(&I, L);
3947  DAG.setRoot(OutChain);
3948 }
3949 
3950 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
3951  SDLoc dl = getCurSDLoc();
3952  ISD::NodeType NT;
3953  switch (I.getOperation()) {
3954  default: llvm_unreachable("Unknown atomicrmw operation");
3955  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
3956  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
3957  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
3958  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
3959  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
3960  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
3961  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
3962  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
3963  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
3964  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
3965  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
3966  }
3967  AtomicOrdering Order = I.getOrdering();
3968  SynchronizationScope Scope = I.getSynchScope();
3969 
3970  SDValue InChain = getRoot();
3971 
3972  SDValue L =
3973  DAG.getAtomic(NT, dl,
3974  getValue(I.getValOperand()).getSimpleValueType(),
3975  InChain,
3977  getValue(I.getValOperand()),
3978  I.getPointerOperand(),
3979  /* Alignment=*/ 0, Order, Scope);
3980 
3981  SDValue OutChain = L.getValue(1);
3982 
3983  setValue(&I, L);
3984  DAG.setRoot(OutChain);
3985 }
3986 
3987 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
3988  SDLoc dl = getCurSDLoc();
3989  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3990  SDValue Ops[3];
3991  Ops[0] = getRoot();
3992  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
3993  TLI.getPointerTy(DAG.getDataLayout()));
3994  Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
3995  TLI.getPointerTy(DAG.getDataLayout()));
3997 }
3998 
3999 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4000  SDLoc dl = getCurSDLoc();
4001  AtomicOrdering Order = I.getOrdering();
4002  SynchronizationScope Scope = I.getSynchScope();
4003 
4004  SDValue InChain = getRoot();
4005 
4006  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4007  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4008 
4009  if (I.getAlignment() < VT.getSizeInBits() / 8)
4010  report_fatal_error("Cannot generate unaligned atomic load");
4011 
4012  MachineMemOperand *MMO =
4014  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4017  VT.getStoreSize(),
4018  I.getAlignment() ? I.getAlignment() :
4019  DAG.getEVTAlignment(VT),
4020  AAMDNodes(), nullptr, Scope, Order);
4021 
4022  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4023  SDValue L =
4024  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4025  getValue(I.getPointerOperand()), MMO);
4026 
4027  SDValue OutChain = L.getValue(1);
4028 
4029  setValue(&I, L);
4030  DAG.setRoot(OutChain);
4031 }
4032 
4033 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4034  SDLoc dl = getCurSDLoc();
4035 
4036  AtomicOrdering Order = I.getOrdering();
4037  SynchronizationScope Scope = I.getSynchScope();
4038 
4039  SDValue InChain = getRoot();
4040 
4041  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4042  EVT VT =
4044 
4045  if (I.getAlignment() < VT.getSizeInBits() / 8)
4046  report_fatal_error("Cannot generate unaligned atomic store");
4047 
4048  SDValue OutChain =
4050  InChain,
4054  Order, Scope);
4055 
4056  DAG.setRoot(OutChain);
4057 }
4058 
4059 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4060 /// node.
4061 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4062  unsigned Intrinsic) {
4063  // Ignore the callsite's attributes. A specific call site may be marked with
4064  // readnone, but the lowering code will expect the chain based on the
4065  // definition.
4066  const Function *F = I.getCalledFunction();
4067  bool HasChain = !F->doesNotAccessMemory();
4068  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4069 
4070  // Build the operand list.
4072  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4073  if (OnlyLoad) {
4074  // We don't need to serialize loads against other loads.
4075  Ops.push_back(DAG.getRoot());
4076  } else {
4077  Ops.push_back(getRoot());
4078  }
4079  }
4080 
4081  // Info is set by getTgtMemInstrinsic
4082  TargetLowering::IntrinsicInfo Info;
4083  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4084  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4085 
4086  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4087  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4088  Info.opc == ISD::INTRINSIC_W_CHAIN)
4089  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4090  TLI.getPointerTy(DAG.getDataLayout())));
4091 
4092  // Add all operands of the call to the operand list.
4093  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4094  SDValue Op = getValue(I.getArgOperand(i));
4095  Ops.push_back(Op);
4096  }
4097 
4098  SmallVector<EVT, 4> ValueVTs;
4099  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4100 
4101  if (HasChain)
4102  ValueVTs.push_back(MVT::Other);
4103 
4104  SDVTList VTs = DAG.getVTList(ValueVTs);
4105 
4106  // Create the node.
4107  SDValue Result;
4108  if (IsTgtIntrinsic) {
4109  // This is target intrinsic that touches memory
4110  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4111  VTs, Ops, Info.memVT,
4112  MachinePointerInfo(Info.ptrVal, Info.offset),
4113  Info.align, Info.vol,
4114  Info.readMem, Info.writeMem, Info.size);
4115  } else if (!HasChain) {
4116  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4117  } else if (!I.getType()->isVoidTy()) {
4118  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4119  } else {
4120  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4121  }
4122 
4123  if (HasChain) {
4124  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4125  if (OnlyLoad)
4126  PendingLoads.push_back(Chain);
4127  else
4128  DAG.setRoot(Chain);
4129  }
4130 
4131  if (!I.getType()->isVoidTy()) {
4132  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4133  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4134  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4135  } else
4136  Result = lowerRangeToAssertZExt(DAG, I, Result);
4137 
4138  setValue(&I, Result);
4139  }
4140 }
4141 
4142 /// GetSignificand - Get the significand and build it into a floating-point
4143 /// number with exponent of 1:
4144 ///
4145 /// Op = (Op & 0x007fffff) | 0x3f800000;
4146 ///
4147 /// where Op is the hexadecimal representation of floating point value.
4148 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4149  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4150  DAG.getConstant(0x007fffff, dl, MVT::i32));
4151  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4152  DAG.getConstant(0x3f800000, dl, MVT::i32));
4153  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4154 }
4155 
4156 /// GetExponent - Get the exponent:
4157 ///
4158 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4159 ///
4160 /// where Op is the hexadecimal representation of floating point value.
4162  const TargetLowering &TLI, const SDLoc &dl) {
4163  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4164  DAG.getConstant(0x7f800000, dl, MVT::i32));
4165  SDValue t1 = DAG.getNode(
4166  ISD::SRL, dl, MVT::i32, t0,
4167  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4168  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4169  DAG.getConstant(127, dl, MVT::i32));
4170  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4171 }
4172 
4173 /// getF32Constant - Get 32-bit floating point constant.
4174 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4175  const SDLoc &dl) {
4176  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4177  MVT::f32);
4178 }
4179 
4181  SelectionDAG &DAG) {
4182  // TODO: What fast-math-flags should be set on the floating-point nodes?
4183 
4184  // IntegerPartOfX = ((int32_t)(t0);
4185  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4186 
4187  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4188  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4189  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4190 
4191  // IntegerPartOfX <<= 23;
4192  IntegerPartOfX = DAG.getNode(
4193  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4195  DAG.getDataLayout())));
4196 
4197  SDValue TwoToFractionalPartOfX;
4198  if (LimitFloatPrecision <= 6) {
4199  // For floating-point precision of 6:
4200  //
4201  // TwoToFractionalPartOfX =
4202  // 0.997535578f +
4203  // (0.735607626f + 0.252464424f * x) * x;
4204  //
4205  // error 0.0144103317, which is 6 bits
4206  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4207  getF32Constant(DAG, 0x3e814304, dl));
4208  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4209  getF32Constant(DAG, 0x3f3c50c8, dl));
4210  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4211  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4212  getF32Constant(DAG, 0x3f7f5e7e, dl));
4213  } else if (LimitFloatPrecision <= 12) {
4214  // For floating-point precision of 12:
4215  //
4216  // TwoToFractionalPartOfX =
4217  // 0.999892986f +
4218  // (0.696457318f +
4219  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4220  //
4221  // error 0.000107046256, which is 13 to 14 bits
4222  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4223  getF32Constant(DAG, 0x3da235e3, dl));
4224  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4225  getF32Constant(DAG, 0x3e65b8f3, dl));
4226  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4227  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4228  getF32Constant(DAG, 0x3f324b07, dl));
4229  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4230  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4231  getF32Constant(DAG, 0x3f7ff8fd, dl));
4232  } else { // LimitFloatPrecision <= 18
4233  // For floating-point precision of 18:
4234  //
4235  // TwoToFractionalPartOfX =
4236  // 0.999999982f +
4237  // (0.693148872f +
4238  // (0.240227044f +
4239  // (0.554906021e-1f +
4240  // (0.961591928e-2f +
4241  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4242  // error 2.47208000*10^(-7), which is better than 18 bits
4243  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4244  getF32Constant(DAG, 0x3924b03e, dl));
4245  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4246  getF32Constant(DAG, 0x3ab24b87, dl));
4247  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4248  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4249  getF32Constant(DAG, 0x3c1d8c17, dl));
4250  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4251  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4252  getF32Constant(DAG, 0x3d634a1d, dl));
4253  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4254  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4255  getF32Constant(DAG, 0x3e75fe14, dl));
4256  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4257  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4258  getF32Constant(DAG, 0x3f317234, dl));
4259  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4260  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4261  getF32Constant(DAG, 0x3f800000, dl));
4262  }
4263 
4264  // Add the exponent into the result in integer domain.
4265  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4266  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4267  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4268 }
4269 
4270 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4271 /// limited-precision mode.
4272 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4273  const TargetLowering &TLI) {
4274  if (Op.getValueType() == MVT::f32 &&
4276 
4277  // Put the exponent in the right bit position for later addition to the
4278  // final result:
4279  //
4280  // #define LOG2OFe 1.4426950f
4281  // t0 = Op * LOG2OFe
4282 
4283  // TODO: What fast-math-flags should be set here?
4284  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4285  getF32Constant(DAG, 0x3fb8aa3b, dl));
4286  return getLimitedPrecisionExp2(t0, dl, DAG);
4287  }
4288 
4289  // No special expansion.
4290  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4291 }
4292 
4293 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4294 /// limited-precision mode.
4295 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4296  const TargetLowering &TLI) {
4297 
4298  // TODO: What fast-math-flags should be set on the floating-point nodes?
4299 
4300  if (Op.getValueType() == MVT::f32 &&
4302  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4303 
4304  // Scale the exponent by log(2) [0.69314718f].
4305  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4306  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4307  getF32Constant(DAG, 0x3f317218, dl));
4308 
4309  // Get the significand and build it into a floating-point number with
4310  // exponent of 1.
4311  SDValue X = GetSignificand(DAG, Op1, dl);
4312 
4313  SDValue LogOfMantissa;
4314  if (LimitFloatPrecision <= 6) {
4315  // For floating-point precision of 6:
4316  //
4317  // LogofMantissa =
4318  // -1.1609546f +
4319  // (1.4034025f - 0.23903021f * x) * x;
4320  //
4321  // error 0.0034276066, which is better than 8 bits
4322  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4323  getF32Constant(DAG, 0xbe74c456, dl));
4324  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4325  getF32Constant(DAG, 0x3fb3a2b1, dl));
4326  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4327  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4328  getF32Constant(DAG, 0x3f949a29, dl));
4329  } else if (LimitFloatPrecision <= 12) {
4330  // For floating-point precision of 12:
4331  //
4332  // LogOfMantissa =
4333  // -1.7417939f +
4334  // (2.8212026f +
4335  // (-1.4699568f +
4336  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4337  //
4338  // error 0.000061011436, which is 14 bits
4339  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4340  getF32Constant(DAG, 0xbd67b6d6, dl));
4341  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4342  getF32Constant(DAG, 0x3ee4f4b8, dl));
4343  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4344  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4345  getF32Constant(DAG, 0x3fbc278b, dl));
4346  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4347  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4348  getF32Constant(DAG, 0x40348e95, dl));
4349  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4350  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4351  getF32Constant(DAG, 0x3fdef31a, dl));
4352  } else { // LimitFloatPrecision <= 18
4353  // For floating-point precision of 18:
4354  //
4355  // LogOfMantissa =
4356  // -2.1072184f +
4357  // (4.2372794f +
4358  // (-3.7029485f +
4359  // (2.2781945f +
4360  // (-0.87823314f +
4361  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4362  //
4363  // error 0.0000023660568, which is better than 18 bits
4364  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4365  getF32Constant(DAG, 0xbc91e5ac, dl));
4366  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4367  getF32Constant(DAG, 0x3e4350aa, dl));
4368  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4369  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4370  getF32Constant(DAG, 0x3f60d3e3, dl));
4371  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4372  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4373  getF32Constant(DAG, 0x4011cdf0, dl));
4374  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4375  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4376  getF32Constant(DAG, 0x406cfd1c, dl));
4377  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4378  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4379  getF32Constant(DAG, 0x408797cb, dl));
4380  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4381  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4382  getF32Constant(DAG, 0x4006dcab, dl));
4383  }
4384 
4385  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4386  }
4387 
4388  // No special expansion.
4389  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4390 }
4391 
4392 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4393 /// limited-precision mode.
4394 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4395  const TargetLowering &TLI) {
4396 
4397  // TODO: What fast-math-flags should be set on the floating-point nodes?
4398 
4399  if (Op.getValueType() == MVT::f32 &&
4401  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4402 
4403  // Get the exponent.
4404  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4405 
4406  // Get the significand and build it into a floating-point number with
4407  // exponent of 1.
4408  SDValue X = GetSignificand(DAG, Op1, dl);
4409 
4410  // Different possible minimax approximations of significand in
4411  // floating-point for various degrees of accuracy over [1,2].
4412  SDValue Log2ofMantissa;
4413  if (LimitFloatPrecision <= 6) {
4414  // For floating-point precision of 6:
4415  //
4416  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4417  //
4418  // error 0.0049451742, which is more than 7 bits
4419  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4420  getF32Constant(DAG, 0xbeb08fe0, dl));
4421  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4422  getF32Constant(DAG, 0x40019463, dl));
4423  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4424  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4425  getF32Constant(DAG, 0x3fd6633d, dl));
4426  } else if (LimitFloatPrecision <= 12) {
4427  // For floating-point precision of 12:
4428  //
4429  // Log2ofMantissa =
4430  // -2.51285454f +
4431  // (4.07009056f +
4432  // (-2.12067489f +
4433  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4434  //
4435  // error 0.0000876136000, which is better than 13 bits
4436  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4437  getF32Constant(DAG, 0xbda7262e, dl));
4438  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4439  getF32Constant(DAG, 0x3f25280b, dl));
4440  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4441  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4442  getF32Constant(DAG, 0x4007b923, dl));
4443  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4444  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4445  getF32Constant(DAG, 0x40823e2f, dl));
4446  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4447  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4448  getF32Constant(DAG, 0x4020d29c, dl));
4449  } else { // LimitFloatPrecision <= 18
4450  // For floating-point precision of 18:
4451  //
4452  // Log2ofMantissa =
4453  // -3.0400495f +
4454  // (6.1129976f +
4455  // (-5.3420409f +
4456  // (3.2865683f +
4457  // (-1.2669343f +
4458  // (0.27515199f -
4459  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4460  //
4461  // error 0.0000018516, which is better than 18 bits
4462  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4463  getF32Constant(DAG, 0xbcd2769e, dl));
4464  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4465  getF32Constant(DAG, 0x3e8ce0b9, dl));
4466  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4467  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4468  getF32Constant(DAG, 0x3fa22ae7, dl));
4469  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4470  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4471  getF32Constant(DAG, 0x40525723, dl));
4472  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4473  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4474  getF32Constant(DAG, 0x40aaf200, dl));
4475  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4476  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4477  getF32Constant(DAG, 0x40c39dad, dl));
4478  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4479  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4480  getF32Constant(DAG, 0x4042902c, dl));
4481  }
4482 
4483  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4484  }
4485 
4486  // No special expansion.
4487  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4488 }
4489 
4490 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4491 /// limited-precision mode.
4492 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4493  const TargetLowering &TLI) {
4494 
4495  // TODO: What fast-math-flags should be set on the floating-point nodes?
4496 
4497  if (Op.getValueType() == MVT::f32 &&
4499  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4500 
4501  // Scale the exponent by log10(2) [0.30102999f].
4502  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4503  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4504  getF32Constant(DAG, 0x3e9a209a, dl));
4505 
4506  // Get the significand and build it into a floating-point number with
4507  // exponent of 1.
4508  SDValue X = GetSignificand(DAG, Op1, dl);
4509 
4510  SDValue Log10ofMantissa;
4511  if (LimitFloatPrecision <= 6) {
4512  // For floating-point precision of 6:
4513  //
4514  // Log10ofMantissa =
4515  // -0.50419619f +
4516  // (0.60948995f - 0.10380950f * x) * x;
4517  //
4518  // error 0.0014886165, which is 6 bits
4519  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4520  getF32Constant(DAG, 0xbdd49a13, dl));
4521  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4522  getF32Constant(DAG, 0x3f1c0789, dl));
4523  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4524  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4525  getF32Constant(DAG, 0x3f011300, dl));
4526  } else if (LimitFloatPrecision <= 12) {
4527  // For floating-point precision of 12:
4528  //
4529  // Log10ofMantissa =
4530  // -0.64831180f +
4531  // (0.91751397f +
4532  // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4533  //
4534  // error 0.00019228036, which is better than 12 bits
4535  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4536  getF32Constant(DAG, 0x3d431f31, dl));
4537  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4538  getF32Constant(DAG, 0x3ea21fb2, dl));
4539  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4540  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4541  getF32Constant(DAG, 0x3f6ae232, dl));
4542  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4543  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4544  getF32Constant(DAG, 0x3f25f7c3, dl));
4545  } else { // LimitFloatPrecision <= 18
4546  // For floating-point precision of 18:
4547  //
4548  // Log10ofMantissa =
4549  // -0.84299375f +
4550  // (1.5327582f +
4551  // (-1.0688956f +
4552  // (0.49102474f +
4553  // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4554  //
4555  // error 0.0000037995730, which is better than 18 bits
4556  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4557  getF32Constant(DAG, 0x3c5d51ce, dl));
4558  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4559  getF32Constant(DAG, 0x3e00685a, dl));
4560  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4561  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4562  getF32Constant(DAG, 0x3efb6798, dl));
4563  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4564  SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4565  getF32Constant(DAG, 0x3f88d192, dl));
4566  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4567  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4568  getF32Constant(DAG, 0x3fc4316c, dl));
4569  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4570  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4571  getF32Constant(DAG, 0x3f57ce70, dl));
4572  }
4573 
4574  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4575  }
4576 
4577  // No special expansion.
4578  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4579 }
4580 
4581 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4582 /// limited-precision mode.
4583 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4584  const TargetLowering &TLI) {
4585  if (Op.getValueType() == MVT::f32 &&
4587  return getLimitedPrecisionExp2(Op, dl, DAG);
4588 
4589  // No special expansion.
4590  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4591 }
4592 
4593 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4594 /// limited-precision mode with x == 10.0f.
4595 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4596  SelectionDAG &DAG, const TargetLowering &TLI) {
4597  bool IsExp10 = false;
4598  if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4600  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4601  APFloat Ten(10.0f);
4602  IsExp10 = LHSC->isExactlyValue(Ten);
4603  }
4604  }
4605 
4606  // TODO: What fast-math-flags should be set on the FMUL node?
4607  if (IsExp10) {
4608  // Put the exponent in the right bit position for later addition to the
4609  // final result:
4610  //
4611  // #define LOG2OF10 3.3219281f
4612  // t0 = Op * LOG2OF10;
4613  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4614  getF32Constant(DAG, 0x40549a78, dl));
4615  return getLimitedPrecisionExp2(t0, dl, DAG);
4616  }
4617 
4618  // No special expansion.
4619  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4620 }
4621 
4622 
4623 /// ExpandPowI - Expand a llvm.powi intrinsic.
4624 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4625  SelectionDAG &DAG) {
4626  // If RHS is a constant, we can expand this out to a multiplication tree,
4627  // otherwise we end up lowering to a call to __powidf2 (for example). When
4628  // optimizing for size, we only want to do this if the expansion would produce
4629  // a small number of multiplies, otherwise we do the full expansion.
4630  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4631  // Get the exponent as a positive value.
4632  unsigned Val = RHSC->getSExtValue();
4633  if ((int)Val < 0) Val = -Val;
4634 
4635  // powi(x, 0) -> 1.0
4636  if (Val == 0)
4637  return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4638 
4639  const Function *F = DAG.getMachineFunction().getFunction();
4640  if (!F->optForSize() ||
4641  // If optimizing for size, don't insert too many multiplies.
4642  // This inserts up to 5 multiplies.
4643  countPopulation(Val) + Log2_32(Val) < 7) {
4644  // We use the simple binary decomposition method to generate the multiply
4645  // sequence. There are more optimal ways to do this (for example,
4646  // powi(x,15) generates one more multiply than it should), but this has
4647  // the benefit of being both really simple and much better than a libcall.
4648  SDValue Res; // Logically starts equal to 1.0
4649  SDValue CurSquare = LHS;
4650  // TODO: Intrinsics should have fast-math-flags that propagate to these
4651  // nodes.
4652  while (Val) {
4653  if (Val & 1) {
4654  if (Res.getNode())
4655  Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4656  else
4657  Res = CurSquare; // 1.0*CurSquare.
4658  }
4659 
4660  CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4661  CurSquare, CurSquare);
4662  Val >>= 1;
4663  }
4664 
4665  // If the original was negative, invert the result, producing 1/(x*x*x).
4666  if (RHSC->getSExtValue() < 0)
4667  Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4668  DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4669  return Res;
4670  }
4671  }
4672 
4673  // Otherwise, expand to a libcall.
4674  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4675 }
4676 
4677 // getUnderlyingArgReg - Find underlying register used for a truncated or
4678 // bitcasted argument.
4679 static unsigned getUnderlyingArgReg(const SDValue &N) {
4680  switch (N.getOpcode()) {
4681  case ISD::CopyFromReg:
4682  return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4683  case ISD::BITCAST:
4684  case ISD::AssertZext:
4685  case ISD::AssertSext:
4686  case ISD::TRUNCATE:
4687  return getUnderlyingArgReg(N.getOperand(0));
4688  default:
4689  return 0;
4690  }
4691 }
4692 
4693 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4694 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
4695 /// At the end of instruction selection, they will be inserted to the entry BB.
4696 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4697  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4698  DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
4699  const Argument *Arg = dyn_cast<Argument>(V);
4700  if (!Arg)
4701  return false;
4702 
4705 
4706  // Ignore inlined function arguments here.
4707  //
4708  // FIXME: Should we be checking DL->inlinedAt() to determine this?
4709  if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4710  return false;
4711 
4713  // Some arguments' frame index is recorded during argument lowering.
4714  if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
4715  Op = MachineOperand::CreateFI(FI);
4716 
4717  if (!Op && N.getNode()) {
4718  unsigned Reg = getUnderlyingArgReg(N);
4719  if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4720  MachineRegisterInfo &RegInfo = MF.getRegInfo();
4721  unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4722  if (PR)
4723  Reg = PR;
4724  }
4725  if (Reg)
4726  Op = MachineOperand::CreateReg(Reg, false);
4727  }
4728 
4729  if (!Op) {
4730  // Check if ValueMap has reg number.
4732  if (VMI != FuncInfo.ValueMap.end())
4733  Op = MachineOperand::CreateReg(VMI->second, false);
4734  }
4735 
4736  if (!Op && N.getNode())
4737  // Check if frame index is available.
4738  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4739  if (FrameIndexSDNode *FINode =
4740  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4741  Op = MachineOperand::CreateFI(FINode->getIndex());
4742 
4743  if (!Op)
4744  return false;
4745 
4746  assert(Variable->isValidLocationForIntrinsic(DL) &&
4747  "Expected inlined-at fields to agree");
4748  if (Op->isReg())
4749  FuncInfo.ArgDbgValues.push_back(
4750  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4751  Op->getReg(), Offset, Variable, Expr));
4752  else
4753  FuncInfo.ArgDbgValues.push_back(
4754  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4755  .addOperand(*Op)
4756  .addImm(Offset)
4757  .addMetadata(Variable)
4758  .addMetadata(Expr));
4759 
4760  return true;
4761 }
4762 
4763 /// Return the appropriate SDDbgValue based on N.
4764 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4765  DILocalVariable *Variable,
4766  DIExpression *Expr, int64_t Offset,
4767  DebugLoc dl,
4768  unsigned DbgSDNodeOrder) {
4769  SDDbgValue *SDV;
4770  auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode());
4771  if (FISDN && Expr->startsWithDeref()) {
4772  // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4773  // stack slot locations as such instead of as indirectly addressed
4774  // locations.
4775  ArrayRef<uint64_t> TrailingElements(Expr->elements_begin() + 1,
4776  Expr->elements_end());
4777  DIExpression *DerefedDIExpr =
4778  DIExpression::get(*DAG.getContext(), TrailingElements);
4779  int FI = FISDN->getIndex();
4780  SDV = DAG.getFrameIndexDbgValue(Variable, DerefedDIExpr, FI, 0, dl,
4781  DbgSDNodeOrder);
4782  } else {
4783  SDV = DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4784  Offset, dl, DbgSDNodeOrder);
4785  }
4786  return SDV;
4787 }
4788 
4789 // VisualStudio defines setjmp as _setjmp
4790 #if defined(_MSC_VER) && defined(setjmp) && \
4791  !defined(setjmp_undefined_for_msvc)
4792 # pragma push_macro("setjmp")
4793 # undef setjmp
4794 # define setjmp_undefined_for_msvc
4795 #endif
4796 
4797 /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If
4798 /// we want to emit this as a call to a named external function, return the name
4799 /// otherwise lower it and return null.
4800 const char *
4801 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4802  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4803  SDLoc sdl = getCurSDLoc();
4804  DebugLoc dl = getCurDebugLoc();
4805  SDValue Res;
4806 
4807  switch (Intrinsic) {
4808  default:
4809  // By default, turn this into a target intrinsic node.
4810  visitTargetIntrinsic(I, Intrinsic);
4811  return nullptr;
4812  case Intrinsic::vastart: visitVAStart(I); return nullptr;
4813  case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4814  case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4815  case Intrinsic::returnaddress:
4818  getValue(I.getArgOperand(0))));
4819  return nullptr;
4820  case Intrinsic::addressofreturnaddress:
4822  TLI.getPointerTy(DAG.getDataLayout())));
4823  return nullptr;
4824  case Intrinsic::frameaddress:
4827  getValue(I.getArgOperand(0))));
4828  return nullptr;
4829  case Intrinsic::read_register: {
4830  Value *Reg = I.getArgOperand(0);
4831  SDValue Chain = getRoot();
4832  SDValue RegName =
4833  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4834  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4835  Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4836  DAG.getVTList(VT, MVT::Other), Chain, RegName);
4837  setValue(&I, Res);
4838  DAG.setRoot(Res.getValue(1));
4839  return nullptr;
4840  }
4841  case Intrinsic::write_register: {
4842  Value *Reg = I.getArgOperand(0);
4843  Value *RegValue = I.getArgOperand(1);
4844  SDValue Chain = getRoot();
4845  SDValue RegName =
4846  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4848  RegName, getValue(RegValue)));
4849  return nullptr;
4850  }
4851  case Intrinsic::setjmp:
4852  return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4853  case Intrinsic::longjmp:
4854  return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4855  case Intrinsic::memcpy: {
4856  SDValue Op1 = getValue(I.getArgOperand(0));
4857  SDValue Op2 = getValue(I.getArgOperand(1));
4858  SDValue Op3 = getValue(I.getArgOperand(2));
4859  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4860  if (!Align)
4861  Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4862  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4863  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4864  SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4865  false, isTC,
4868  updateDAGForMaybeTailCall(MC);
4869  return nullptr;
4870  }
4871  case Intrinsic::memset: {
4872  SDValue Op1 = getValue(I.getArgOperand(0));
4873  SDValue Op2 = getValue(I.getArgOperand(1));
4874  SDValue Op3 = getValue(I.getArgOperand(2));
4875  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4876  if (!Align)
4877  Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4878  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4879  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4880  SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4881  isTC, MachinePointerInfo(I.getArgOperand(0)));
4882  updateDAGForMaybeTailCall(MS);
4883  return nullptr;
4884  }
4885  case Intrinsic::memmove: {
4886  SDValue Op1 = getValue(I.getArgOperand(0));
4887  SDValue Op2 = getValue(I.getArgOperand(1));
4888  SDValue Op3 = getValue(I.getArgOperand(2));
4889  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4890  if (!Align)
4891  Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4892  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4893  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4894  SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4895  isTC, MachinePointerInfo(I.getArgOperand(0)),
4897  updateDAGForMaybeTailCall(MM);
4898  return nullptr;
4899  }
4900  case Intrinsic::memcpy_element_atomic: {
4901  SDValue Dst = getValue(I.getArgOperand(0));
4902  SDValue Src = getValue(I.getArgOperand(1));
4903  SDValue NumElements = getValue(I.getArgOperand(2));
4904  SDValue ElementSize = getValue(I.getArgOperand(3));
4905 
4906  // Emit a library call.
4910  Entry.Node = Dst;
4911  Args.push_back(Entry);
4912 
4913  Entry.Node = Src;
4914  Args.push_back(Entry);
4915 
4916  Entry.Ty = I.getArgOperand(2)->getType();
4917  Entry.Node = NumElements;
4918  Args.push_back(Entry);
4919 
4920  Entry.Ty = Type::getInt32Ty(*DAG.getContext());
4921  Entry.Node = ElementSize;
4922  Args.push_back(Entry);
4923 
4924  uint64_t ElementSizeConstant =
4925  cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4926  RTLIB::Libcall LibraryCall =
4927  RTLIB::getMEMCPY_ELEMENT_ATOMIC(ElementSizeConstant);
4928  if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
4929  report_fatal_error("Unsupported element size");
4930 
4932  CLI.setDebugLoc(sdl)
4933  .setChain(getRoot())
4934  .setCallee(TLI.getLibcallCallingConv(LibraryCall),
4937  TLI.getLibcallName(LibraryCall),
4938  TLI.getPointerTy(DAG.getDataLayout())),
4939  std::move(Args));
4940 
4941  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4942  DAG.setRoot(CallResult.second);
4943  return nullptr;
4944  }
4945  case Intrinsic::dbg_declare: {
4946  const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
4947  DILocalVariable *Variable = DI.getVariable();
4948  DIExpression *Expression = DI.getExpression();
4949  const Value *Address = DI.getAddress();
4950  assert(Variable && "Missing variable");
4951  if (!Address) {
4952  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4953  return nullptr;
4954  }
4955 
4956  // Check if address has undef value.
4957  if (isa<UndefValue>(Address) ||
4958  (Address->use_empty() && !isa<Argument>(Address))) {
4959  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4960  return nullptr;
4961  }
4962 
4963  SDValue &N = NodeMap[Address];
4964  if (!N.getNode() && isa<Argument>(Address))
4965  // Check unused arguments map.
4966  N = UnusedArgNodeMap[Address];
4967  SDDbgValue *SDV;
4968  if (N.getNode()) {
4969  if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
4970  Address = BCI->getOperand(0);
4971  // Parameters are handled specially.
4972  bool isParameter = Variable->isParameter() || isa<Argument>(Address);
4973  auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
4974  if (isParameter && FINode) {
4975  // Byval parameter. We have a frame index at this point.
4976  SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
4977  FINode->getIndex(), 0, dl, SDNodeOrder);
4978  } else if (isa<Argument>(Address)) {
4979  // Address is an argument, so try to emit its dbg value using
4980  // virtual register info from the FuncInfo.ValueMap.
4981  EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
4982  N);
4983  return nullptr;
4984  } else {
4985  SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
4986  true, 0, dl, SDNodeOrder);
4987  }
4988  DAG.AddDbgValue(SDV, N.getNode(), isParameter);
4989  } else {
4990  // If Address is an argument then try to emit its dbg value using
4991  // virtual register info from the FuncInfo.ValueMap.
4992  if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
4993  N)) {
4994  // If variable is pinned by a alloca in dominating bb then
4995  // use StaticAllocaMap.
4996  if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
4997  if (AI->getParent() != DI.getParent()) {
4999  FuncInfo.StaticAllocaMap.find(AI);
5000  if (SI != FuncInfo.StaticAllocaMap.end()) {
5001  SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
5002  0, dl, SDNodeOrder);
5003  DAG.AddDbgValue(SDV, nullptr, false);
5004  return nullptr;
5005  }
5006  }
5007  }
5008  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5009  }
5010  }
5011  return nullptr;
5012  }
5013  case Intrinsic::dbg_value: {
5014  const DbgValueInst &DI = cast<DbgValueInst>(I);
5015  assert(DI.getVariable() && "Missing variable");
5016 
5017  DILocalVariable *Variable = DI.getVariable();
5018  DIExpression *Expression = DI.getExpression();
5019  uint64_t Offset = DI.getOffset();
5020  const Value *V = DI.getValue();
5021  if (!V)
5022  return nullptr;
5023 
5024  SDDbgValue *SDV;
5025  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
5026  SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
5027  SDNodeOrder);
5028  DAG.AddDbgValue(SDV, nullptr, false);
5029  } else {
5030  // Do not use getValue() in here; we don't want to generate code at
5031  // this point if it hasn't been done yet.
5032  SDValue N = NodeMap[V];
5033  if (!N.getNode() && isa<Argument>(V))
5034  // Check unused arguments map.
5035  N = UnusedArgNodeMap[V];
5036  if (N.getNode()) {
5037  if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
5038  false, N)) {
5039  SDV = getDbgValue(N, Variable, Expression, Offset, dl, SDNodeOrder);
5040  DAG.AddDbgValue(SDV, N.getNode(), false);
5041  }
5042  } else if (!V->use_empty() ) {
5043  // Do not call getValue(V) yet, as we don't want to generate code.
5044  // Remember it for later.
5045  DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5046  DanglingDebugInfoMap[V] = DDI;
5047  } else {
5048  // We may expand this to cover more cases. One case where we have no
5049  // data available is an unreferenced parameter.
5050  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5051  }
5052  }
5053 
5054  // Build a debug info table entry.
5055  if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
5056  V = BCI->getOperand(0);
5057  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
5058  // Don't handle byval struct arguments or VLAs, for example.
5059  if (!AI) {
5060  DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n");
5061  DEBUG(dbgs() << " Last seen at:\n " << *V << "\n");
5062  return nullptr;
5063  }
5065  FuncInfo.StaticAllocaMap.find(AI);
5066  if (SI == FuncInfo.StaticAllocaMap.end())
5067  return nullptr; // VLAs.
5068  return nullptr;
5069  }
5070 
5071  case Intrinsic::eh_typeid_for: {
5072  // Find the type id for the given typeinfo.
5074  unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5075  Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5076  setValue(&I, Res);
5077  return nullptr;
5078  }
5079 
5080  case Intrinsic::eh_return_i32:
5081  case Intrinsic::eh_return_i64:
5084  MVT::Other,
5085  getControlRoot(),
5086  getValue(I.getArgOperand(0)),
5087  getValue(I.getArgOperand(1))));
5088  return nullptr;
5089  case Intrinsic::eh_unwind_init:
5091  return nullptr;
5092  case Intrinsic::eh_dwarf_cfa: {
5095  getValue(I.getArgOperand(0))));
5096  return nullptr;
5097  }
5098  case Intrinsic::eh_sjlj_callsite: {
5101  assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
5102  assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
5103 
5104  MMI.setCurrentCallSite(CI->getZExtValue());
5105  return nullptr;
5106  }
5107  case Intrinsic::eh_sjlj_functioncontext: {
5108  // Get and store the index of the function context.
5110  AllocaInst *FnCtx =
5111  cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5112  int FI = FuncInfo.StaticAllocaMap[FnCtx];
5113  MFI.setFunctionContextIndex(FI);
5114  return nullptr;
5115  }
5116  case Intrinsic::eh_sjlj_setjmp: {
5117  SDValue Ops[2];
5118  Ops[0] = getRoot();
5119  Ops[1] = getValue(I.getArgOperand(0));
5121  DAG.getVTList(MVT::i32, MVT::Other), Ops);
5122  setValue(&I, Op.getValue(0));
5123  DAG.setRoot(Op.getValue(1));
5124  return nullptr;
5125  }
5126  case Intrinsic::eh_sjlj_longjmp: {
5128  getRoot(), getValue(I.getArgOperand(0))));
5129  return nullptr;
5130  }
5131  case Intrinsic::eh_sjlj_setup_dispatch: {
5133  getRoot()));
5134  return nullptr;
5135  }
5136 
5137  case Intrinsic::masked_gather:
5138  visitMaskedGather(I);
5139  return nullptr;
5140  case Intrinsic::masked_load:
5141  visitMaskedLoad(I);
5142  return nullptr;
5143  case Intrinsic::masked_scatter:
5144  visitMaskedScatter(I);
5145  return nullptr;
5146  case Intrinsic::masked_store:
5147  visitMaskedStore(I);
5148  return nullptr;
5149  case Intrinsic::masked_expandload:
5150  visitMaskedLoad(I, true /* IsExpanding */);
5151  return nullptr;
5152  case Intrinsic::masked_compressstore:
5153  visitMaskedStore(I, true /* IsCompressing */);
5154  return nullptr;
5155  case Intrinsic::x86_mmx_pslli_w:
5156  case Intrinsic::x86_mmx_pslli_d:
5157  case Intrinsic::x86_mmx_pslli_q:
5158  case Intrinsic::x86_mmx_psrli_w:
5159  case Intrinsic::x86_mmx_psrli_d:
5160  case Intrinsic::x86_mmx_psrli_q:
5161  case Intrinsic::x86_mmx_psrai_w:
5162  case Intrinsic::x86_mmx_psrai_d: {
5163  SDValue ShAmt = getValue(I.getArgOperand(1));
5164  if (isa<ConstantSDNode>(ShAmt)) {
5165  visitTargetIntrinsic(I, Intrinsic);
5166  return nullptr;
5167  }
5168  unsigned NewIntrinsic = 0;
5169  EVT ShAmtVT = MVT::v2i32;
5170  switch (Intrinsic) {
5171  case Intrinsic::x86_mmx_pslli_w:
5172  NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5173  break;
5174  case Intrinsic::x86_mmx_pslli_d:
5175  NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5176  break;
5177  case Intrinsic::x86_mmx_pslli_q:
5178  NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5179  break;
5180  case Intrinsic::x86_mmx_psrli_w:
5181  NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5182  break;
5183  case Intrinsic::x86_mmx_psrli_d:
5184  NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5185  break;
5186  case Intrinsic::x86_mmx_psrli_q:
5187  NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5188  break;
5189  case Intrinsic::x86_mmx_psrai_w:
5190  NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5191  break;
5192  case Intrinsic::x86_mmx_psrai_d:
5193  NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5194  break;
5195  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5196  }
5197 
5198  // The vector shift intrinsics with scalars uses 32b shift amounts but
5199  // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5200  // to be zero.
5201  // We must do this early because v2i32 is not a legal type.
5202  SDValue ShOps[2];
5203  ShOps[0] = ShAmt;
5204  ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5205  ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
5206  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5207  ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5208  Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5209  DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5210  getValue(I.getArgOperand(0)), ShAmt);
5211  setValue(&I, Res);
5212  return nullptr;
5213  }
5214  case Intrinsic::powi:
5215  setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5216  getValue(I.getArgOperand(1)), DAG));
5217  return nullptr;
5218  case Intrinsic::log:
5219  setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5220  return nullptr;
5221  case Intrinsic::log2:
5222  setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5223  return nullptr;
5224  case Intrinsic::log10:
5225  setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5226  return nullptr;
5227  case Intrinsic::exp:
5228  setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5229  return nullptr;
5230  case Intrinsic::exp2:
5231  setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5232  return nullptr;
5233  case Intrinsic::pow:
5234  setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5235  getValue(I.getArgOperand(1)), DAG, TLI));
5236  return nullptr;
5237  case Intrinsic::sqrt:
5238  case Intrinsic::fabs:
5239  case Intrinsic::sin:
5240  case Intrinsic::cos:
5241  case Intrinsic::floor:
5242  case Intrinsic::ceil:
5243  case Intrinsic::trunc:
5244  case Intrinsic::rint:
5245  case Intrinsic::nearbyint:
5246  case Intrinsic::round:
5247  case Intrinsic::canonicalize: {
5248  unsigned Opcode;
5249  switch (Intrinsic) {
5250  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5251  case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5252  case Intrinsic::fabs: Opcode = ISD::FABS; break;
5253  case Intrinsic::sin: Opcode = ISD::FSIN; break;
5254  case Intrinsic::cos: Opcode = ISD::FCOS; break;
5255  case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5256  case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5257  case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5258  case Intrinsic::rint: Opcode = ISD::FRINT; break;
5259  case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5260  case Intrinsic::round: Opcode = ISD::FROUND; break;
5261  case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5262  }
5263 
5264  setValue(&I, DAG.getNode(Opcode, sdl,
5265  getValue(I.getArgOperand(0)).getValueType(),
5266  getValue(I.getArgOperand(0))));
5267  return nullptr;
5268  }
5269  case Intrinsic::minnum: {
5270  auto VT = getValue(I.getArgOperand(0)).getValueType();
5271  unsigned Opc =
5273  ? ISD::FMINNAN
5274  : ISD::FMINNUM;
5275  setValue(&I, DAG.getNode(Opc, sdl, VT,
5276  getValue(I.getArgOperand(0)),
5277  getValue(I.getArgOperand(1))));
5278  return nullptr;
5279  }
5280  case Intrinsic::maxnum: {
5281  auto VT = getValue(I.getArgOperand(0)).getValueType();
5282  unsigned Opc =
5284  ? ISD::FMAXNAN
5285  : ISD::FMAXNUM;
5286  setValue(&I, DAG.getNode(Opc, sdl, VT,
5287  getValue(I.getArgOperand(0)),
5288  getValue(I.getArgOperand(1))));
5289  return nullptr;
5290  }
5291  case Intrinsic::copysign:
5293  getValue(I.getArgOperand(0)).getValueType(),
5294  getValue(I.getArgOperand(0)),
5295  getValue(I.getArgOperand(1))));
5296  return nullptr;
5297  case Intrinsic::fma:
5298  setValue(&I, DAG.getNode(ISD::FMA, sdl,
5299  getValue(I.getArgOperand(0)).getValueType(),
5300  getValue(I.getArgOperand(0)),
5301  getValue(I.getArgOperand(1)),
5302  getValue(I.getArgOperand(2))));
5303  return nullptr;
5304  case Intrinsic::fmuladd: {
5305  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5306  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5307  TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5308  setValue(&I, DAG.getNode(ISD::FMA, sdl,
5309  getValue(I.getArgOperand(0)).getValueType(),
5310  getValue(I.getArgOperand(0)),
5311  getValue(I.getArgOperand(1)),
5312  getValue(I.getArgOperand(2))));
5313  } else {
5314  // TODO: Intrinsic calls should have fast-math-flags.
5315  SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5316  getValue(I.getArgOperand(0)).getValueType(),
5317  getValue(I.getArgOperand(0)),
5318  getValue(I.getArgOperand(1)));
5319  SDValue Add = DAG.getNode(ISD::FADD, sdl,
5320  getValue(I.getArgOperand(0)).getValueType(),
5321  Mul,
5322  getValue(I.getArgOperand(2)));
5323  setValue(&I, Add);
5324  }
5325  return nullptr;
5326  }
5327  case Intrinsic::convert_to_fp16:
5330  getValue(I.getArgOperand(0)),
5331  DAG.getTargetConstant(0, sdl,
5332  MVT::i32))));
5333  return nullptr;
5334  case Intrinsic::convert_from_fp16:
5336  TLI.getValueType(DAG.getDataLayout(), I.getType()),
5338  getValue(I.getArgOperand(0)))));
5339  return nullptr;
5340  case Intrinsic::pcmarker: {
5341  SDValue Tmp = getValue(I.getArgOperand(0));
5343  return nullptr;
5344  }
5345  case Intrinsic::readcyclecounter: {
5346  SDValue Op = getRoot();
5347  Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5349  setValue(&I, Res);
5350  DAG.setRoot(Res.getValue(1));
5351  return nullptr;
5352  }
5353  case Intrinsic::bitreverse:
5355  getValue(I.getArgOperand(0)).getValueType(),
5356  getValue(I.getArgOperand(0))));
5357  return nullptr;
5358  case Intrinsic::bswap:
5359  setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5360  getValue(I.getArgOperand(0)).getValueType(),
5361  getValue(I.getArgOperand(0))));
5362  return nullptr;
5363  case Intrinsic::cttz: {
5364  SDValue Arg = getValue(I.getArgOperand(0));
5365  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5366  EVT Ty = Arg.getValueType();
5367  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5368  sdl, Ty, Arg));
5369  return nullptr;
5370  }
5371  case Intrinsic::ctlz: {
5372  SDValue Arg = getValue(I.getArgOperand(0));
5373  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5374  EVT Ty = Arg.getValueType();
5375  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5376  sdl, Ty, Arg));
5377  return nullptr;
5378  }
5379  case Intrinsic::ctpop: {
5380  SDValue Arg = getValue(I.getArgOperand(0));
5381  EVT Ty = Arg.getValueType();
5382  setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5383  return nullptr;
5384  }
5385  case Intrinsic::stacksave: {
5386  SDValue Op = getRoot();
5387  Res = DAG.getNode(
5388  ISD::STACKSAVE, sdl,
5390  setValue(&I, Res);
5391  DAG.setRoot(Res.getValue(1));
5392  return nullptr;
5393  }
5394  case Intrinsic::stackrestore: {
5395  Res = getValue(I.getArgOperand(0));
5397  return nullptr;
5398  }
5399  case Intrinsic::get_dynamic_area_offset: {
5400  SDValue Op = getRoot();
5401  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5402  EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5403  // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5404  // target.
5405  if (PtrTy != ResTy)
5406  report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5407  " intrinsic!");
5409  Op);
5410  DAG.setRoot(Op);
5411  setValue(&I, Res);
5412  return nullptr;
5413  }
5414  case Intrinsic::stackguard: {
5415  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5417  const Module &M = *MF.getFunction()->getParent();
5418  SDValue Chain = getRoot();
5419  if (TLI.useLoadStackGuardNode()) {
5420  Res = getLoadStackGuard(DAG, sdl, Chain);
5421  } else {
5422  const Value *Global = TLI.getSDagStackGuard(M);
5423  unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5424  Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5425  MachinePointerInfo(Global, 0), Align,
5427  }
5428  DAG.setRoot(Chain);
5429  setValue(&I, Res);
5430  return nullptr;
5431  }
5432  case Intrinsic::stackprotector: {
5433  // Emit code into the DAG to store the stack guard onto the stack.
5435  MachineFrameInfo &MFI = MF.getFrameInfo();
5436  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5437  SDValue Src, Chain = getRoot();
5438 
5439  if (TLI.useLoadStackGuardNode())
5440  Src = getLoadStackGuard(DAG, sdl, Chain);
5441  else
5442  Src = getValue(I.getArgOperand(0)); // The guard's value.
5443 
5444  AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5445 
5446  int FI = FuncInfo.StaticAllocaMap[Slot];
5447  MFI.setStackProtectorIndex(FI);
5448 
5449  SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5450 
5451  // Store the stack protector onto the stack.
5452  Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5453  DAG.getMachineFunction(), FI),
5454  /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5455  setValue(&I, Res);
5456  DAG.setRoot(Res);
5457  return nullptr;
5458  }
5459  case Intrinsic::objectsize: {
5460  // If we don't know by now, we're never going to know.
5462 
5463  assert(CI && "Non-constant type in __builtin_object_size?");
5464 
5465  SDValue Arg = getValue(I.getCalledValue());
5466  EVT Ty = Arg.getValueType();
5467 
5468  if (CI->isZero())
5469  Res = DAG.getConstant(-1ULL, sdl, Ty);
5470  else
5471  Res = DAG.getConstant(0, sdl, Ty);
5472 
5473  setValue(&I, Res);
5474  return nullptr;
5475  }
5476  case Intrinsic::annotation:
5477  case Intrinsic::ptr_annotation:
5478  case Intrinsic::invariant_group_barrier:
5479  // Drop the intrinsic, but forward the value
5480  setValue(&I, getValue(I.getOperand(0)));
5481  return nullptr;
5482  case Intrinsic::assume:
5483  case Intrinsic::var_annotation:
5484  // Discard annotate attributes and assumptions
5485  return nullptr;
5486 
5487  case Intrinsic::init_trampoline: {
5488  const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5489 
5490  SDValue Ops[6];
5491  Ops[0] = getRoot();
5492  Ops[1] = getValue(I.getArgOperand(0));
5493  Ops[2] = getValue(I.getArgOperand(1));
5494  Ops[3] = getValue(I.getArgOperand(2));
5495  Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5496  Ops[5] = DAG.getSrcValue(F);
5497 
5498  Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5499 
5500  DAG.setRoot(Res);
5501  return nullptr;
5502  }
5503  case Intrinsic::adjust_trampoline: {
5506  getValue(I.getArgOperand(0))));
5507  return nullptr;
5508  }
5509  case Intrinsic::gcroot: {
5511  const Function *F = MF.getFunction();
5512  (void)F;
5513  assert(F->hasGC() &&
5514  "only valid in functions with gc specified, enforced by Verifier");
5515  assert(GFI && "implied by previous");
5516  const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5517  const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5518 
5519  FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5520  GFI->addStackRoot(FI->getIndex(), TypeMap);
5521  return nullptr;
5522  }
5523  case Intrinsic::gcread:
5524  case Intrinsic::gcwrite:
5525  llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
5526  case Intrinsic::flt_rounds:
5528  return nullptr;
5529 
5530  case Intrinsic::expect: {
5531  // Just replace __builtin_expect(exp, c) with EXP.
5532  setValue(&I, getValue(I.getArgOperand(0)));
5533  return nullptr;
5534  }
5535 
5536  case Intrinsic::debugtrap:
5537  case Intrinsic::trap: {
5539  I.getAttributes()
5540  .getAttribute(AttributeSet::FunctionIndex, "trap-func-name")
5541  .getValueAsString();
5542  if (TrapFuncName.empty()) {
5543  ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5545  DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5546  return nullptr;
5547  }
5549 
5551  CLI.setDebugLoc(sdl).setChain(getRoot()).setCallee(
5552  CallingConv::C, I.getType(),
5553  DAG.getExternalSymbol(TrapFuncName.data(),
5554  TLI.getPointerTy(DAG.getDataLayout())),
5555  std::move(Args));
5556 
5557  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5558  DAG.setRoot(Result.second);
5559  return nullptr;
5560  }
5561 
5562  case Intrinsic::uadd_with_overflow:
5563  case Intrinsic::sadd_with_overflow:
5564  case Intrinsic::usub_with_overflow:
5565  case Intrinsic::ssub_with_overflow:
5566  case Intrinsic::umul_with_overflow:
5567  case Intrinsic::smul_with_overflow: {
5568  ISD::NodeType Op;
5569  switch (Intrinsic) {
5570  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5571  case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5572  case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5573  case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5574  case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5575  case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5576  case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5577  }
5578  SDValue Op1 = getValue(I.getArgOperand(0));
5579  SDValue Op2 = getValue(I.getArgOperand(1));
5580 
5581  SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5582  setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5583  return nullptr;
5584  }
5585  case Intrinsic::prefetch: {
5586  SDValue Ops[5];
5587  unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5588  Ops[0] = getRoot();
5589  Ops[1] = getValue(I.getArgOperand(0));
5590  Ops[2] = getValue(I.getArgOperand(1));
5591  Ops[3] = getValue(I.getArgOperand(2));
5592  Ops[4] = getValue(I.getArgOperand(3));
5594  DAG.getVTList(MVT::Other), Ops,
5595  EVT::getIntegerVT(*Context, 8),
5597  0, /* align */
5598  false, /* volatile */
5599  rw==0, /* read */
5600  rw==1)); /* write */
5601  return nullptr;
5602  }
5603  case Intrinsic::lifetime_start:
5604  case Intrinsic::lifetime_end: {
5605  bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5606  // Stack coloring is not enabled in O0, discard region information.
5607  if (TM.getOptLevel() == CodeGenOpt::None)
5608  return nullptr;
5609 
5610  SmallVector<Value *, 4> Allocas;
5611  GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5612 
5613  for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5614  E = Allocas.end(); Object != E; ++Object) {
5615  AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5616 
5617  // Could not find an Alloca.
5618  if (!LifetimeObject)
5619  continue;
5620 
5621  // First check that the Alloca is static, otherwise it won't have a
5622  // valid frame index.
5623  auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5624  if (SI == FuncInfo.StaticAllocaMap.end())
5625  return nullptr;
5626 
5627  int FI = SI->second;
5628 
5629  SDValue Ops[2];
5630  Ops[0] = getRoot();
5631  Ops[1] =
5632  DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout()), true);
5633  unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5634 
5635  Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
5636  DAG.setRoot(Res);
5637  }
5638  return nullptr;
5639  }
5640  case Intrinsic::invariant_start:
5641  // Discard region information.
5643  return nullptr;
5644  case Intrinsic::invariant_end:
5645  // Discard region information.
5646  return nullptr;
5647  case Intrinsic::clear_cache:
5648  return TLI.getClearCacheBuiltinName();
5649  case Intrinsic::donothing:
5650  // ignore
5651  return nullptr;
5652  case Intrinsic::experimental_stackmap: {
5653  visitStackmap(I);
5654  return nullptr;
5655  }
5656  case Intrinsic::experimental_patchpoint_void:
5657  case Intrinsic::experimental_patchpoint_i64: {
5658  visitPatchpoint(&I);
5659  return nullptr;
5660  }
5661  case Intrinsic::experimental_gc_statepoint: {
5663  return nullptr;
5664  }
5665  case Intrinsic::experimental_gc_result: {
5666  visitGCResult(cast<GCResultInst>(I));
5667  return nullptr;
5668  }
5669  case Intrinsic::experimental_gc_relocate: {
5670  visitGCRelocate(cast<GCRelocateInst>(I));
5671  return nullptr;
5672  }
5673  case Intrinsic::instrprof_increment:
5674  llvm_unreachable("instrprof failed to lower an increment");
5675  case Intrinsic::instrprof_value_profile:
5676  llvm_unreachable("instrprof failed to lower a value profiling call");
5677  case Intrinsic::localescape: {
5679  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5680 
5681  // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
5682  // is the same on all targets.
5683  for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
5684  Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
5685  if (isa<ConstantPointerNull>(Arg))
5686  continue; // Skip null pointers. They represent a hole in index space.
5687  AllocaInst *Slot = cast<AllocaInst>(Arg);
5688  assert(FuncInfo.StaticAllocaMap.count(Slot) &&
5689  "can only escape static allocas");
5690  int FI = FuncInfo.StaticAllocaMap[Slot];
5691  MCSymbol *FrameAllocSym =
5695  TII->get(TargetOpcode::LOCAL_ESCAPE))
5696  .addSym(FrameAllocSym)
5697  .addFrameIndex(FI);
5698  }
5699 
5700  return nullptr;
5701  }
5702 
5703  case Intrinsic::localrecover: {
5704  // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5706  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5707 
5708  // Get the symbol that defines the frame offset.
5709  auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5710  auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5711  unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
5712  MCSymbol *FrameAllocSym =
5714  GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
5715 
5716  // Create a MCSymbol for the label to avoid any target lowering
5717  // that would make this PC relative.
5718  SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5719  SDValue OffsetVal =
5720  DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5721 
5722  // Add the offset to the FP.
5723  Value *FP = I.getArgOperand(1);
5724  SDValue FPVal = getValue(FP);
5725  SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5726  setValue(&I, Add);
5727 
5728  return nullptr;
5729  }
5730 
5731  case Intrinsic::eh_exceptionpointer:
5732  case Intrinsic::eh_exceptioncode: {
5733  // Get the exception pointer vreg, copy from it, and resize it to fit.
5734  const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
5735  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5736  const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5737  unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
5738  SDValue N =
5739  DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5740  if (Intrinsic == Intrinsic::eh_exceptioncode)
5742  setValue(&I, N);
5743  return nullptr;
5744  }
5745 
5746  case Intrinsic::experimental_deoptimize:
5747  LowerDeoptimizeCall(&I);
5748  return nullptr;
5749  }
5750 }
5751 
5752 std::pair<SDValue, SDValue>
5754  const BasicBlock *EHPadBB) {
5756  MachineModuleInfo &MMI = MF.getMMI();
5757  MCSymbol *BeginLabel = nullptr;
5758 
5759  if (EHPadBB) {
5760  // Insert a label before the invoke call to mark the try range. This can be
5761  // used to detect deletion of the invoke via the MachineModuleInfo.
5762  BeginLabel = MMI.getContext().createTempSymbol();
5763 
5764  // For SjLj, keep track of which landing pads go with which invokes
5765  // so as to maintain the ordering of pads in the LSDA.
5766  unsigned CallSiteIndex = MMI.getCurrentCallSite();
5767  if (CallSiteIndex) {
5768  MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
5769  LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
5770 
5771  // Now that the call site is handled, stop tracking it.
5772  MMI.setCurrentCallSite(0);
5773  }
5774 
5775  // Both PendingLoads and PendingExports must be flushed here;
5776  // this call might not return.
5777  (void)getRoot();
5778  DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
5779 
5780  CLI.setChain(getRoot());
5781  }
5782  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5783  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5784 
5785  assert((CLI.IsTailCall || Result.second.getNode()) &&
5786  "Non-null chain expected with non-tail call!");
5787  assert((Result.second.getNode() || !Result.first.getNode()) &&
5788  "Null value expected with tail call!");
5789 
5790  if (!Result.second.getNode()) {
5791  // As a special case, a null chain means that a tail call has been emitted
5792  // and the DAG root is already updated.
5793  HasTailCall = true;
5794 
5795  // Since there's no actual continuation from this block, nothing can be
5796  // relying on us setting vregs for them.
5797  PendingExports.clear();
5798  } else {
5799  DAG.setRoot(Result.second);
5800  }
5801 
5802  if (EHPadBB) {
5803  // Insert a label at the end of the invoke call to mark the try range. This
5804  // can be used to detect deletion of the invoke via the MachineModuleInfo.
5805  MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
5806  DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
5807 
5808  // Inform MachineModuleInfo of range.
5809  if (MF.hasEHFunclets()) {
5810  assert(CLI.CS);
5812  EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS->getInstruction()),
5813  BeginLabel, EndLabel);
5814  } else {
5815  MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
5816  }
5817  }
5818 
5819  return Result;
5820 }
5821 
5823  bool isTailCall,
5824  const BasicBlock *EHPadBB) {
5825  auto &DL = DAG.getDataLayout();
5826  FunctionType *FTy = CS.getFunctionType();
5827  Type *RetTy = CS.getType();
5828 
5831  Args.reserve(CS.arg_size());
5832 
5833  const Value *SwiftErrorVal = nullptr;
5834  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5835 
5836  // We can't tail call inside a function with a swifterror argument. Lowering
5837  // does not support this yet. It would have to move into the swifterror
5838  // register before the call.
5839  auto *Caller = CS.getInstruction()->getParent()->getParent();
5840  if (TLI.supportSwiftError() &&
5841  Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
5842  isTailCall = false;
5843 
5844  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
5845  i != e; ++i) {
5846  const Value *V = *i;
5847 
5848  // Skip empty types
5849  if (V->getType()->isEmptyTy())
5850  continue;
5851 
5852  SDValue ArgNode = getValue(V);
5853  Entry.Node = ArgNode; Entry.Ty = V->getType();
5854 
5855  // Skip the first return-type Attribute to get to params.
5856  Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
5857 
5858  // Use swifterror virtual register as input to the call.
5859  if (Entry.isSwiftError && TLI.supportSwiftError()) {
5860  SwiftErrorVal = V;
5861  // We find the virtual register for the actual swifterror argument.
5862  // Instead of using the Value, we use the virtual register instead.
5863  Entry.Node =
5865  EVT(TLI.getPointerTy(DL)));
5866  }
5867 
5868  Args.push_back(Entry);
5869 
5870  // If we have an explicit sret argument that is an Instruction, (i.e., it
5871  // might point to function-local memory), we can't meaningfully tail-call.
5872  if (Entry.isSRet && isa<Instruction>(V))
5873  isTailCall = false;
5874  }
5875 
5876  // Check if target-independent constraints permit a tail call here.
5877  // Target-dependent constraints are checked within TLI->LowerCallTo.
5878  if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
5879  isTailCall = false;
5880 
5881  // Disable tail calls if there is an swifterror argument. Targets have not
5882  // been updated to support tail calls.
5883  if (TLI.supportSwiftError() && SwiftErrorVal)
5884  isTailCall = false;
5885 
5887  CLI.setDebugLoc(getCurSDLoc())
5888  .setChain(getRoot())
5889  .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
5890  .setTailCall(isTailCall)
5891  .setConvergent(CS.isConvergent());
5892  std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
5893 
5894  if (Result.first.getNode()) {
5895  const Instruction *Inst = CS.getInstruction();
5896  Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
5897  setValue(Inst, Result.first);
5898  }
5899 
5900  // The last element of CLI.InVals has the SDValue for swifterror return.
5901  // Here we copy it to a virtual register and update SwiftErrorMap for
5902  // book-keeping.
5903  if (SwiftErrorVal && TLI.supportSwiftError()) {
5904  // Get the last element of InVals.
5905  SDValue Src = CLI.InVals.back();
5906  const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
5907  unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
5908  SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
5909  // We update the virtual register for the actual swifterror argument.
5910  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
5911  DAG.setRoot(CopyNode);
5912  }
5913 }
5914 
5915 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
5916 /// value is equal or not-equal to zero.
5918  for (const User *U : V->users()) {
5919  if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
5920  if (IC->isEquality())
5921  if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
5922  if (C->isNullValue())
5923  continue;
5924  // Unknown instruction.
5925  return false;
5926  }
5927  return true;
5928 }
5929 
5930 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
5931  Type *LoadTy,
5932  SelectionDAGBuilder &Builder) {
5933 
5934  // Check to see if this load can be trivially constant folded, e.g. if the
5935  // input is from a string literal.
5936  if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
5937  // Cast pointer to the type we really want to load.
5938  LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
5939  PointerType::getUnqual(LoadTy));
5940 
5941  if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
5942  const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
5943  return Builder.getValue(LoadCst);
5944  }
5945 
5946  // Otherwise, we have to emit the load. If the pointer is to unfoldable but
5947  // still constant memory, the input chain can be the entry node.
5948  SDValue Root;
5949  bool ConstantMemory = false;
5950 
5951  // Do not serialize (non-volatile) loads of constant memory with anything.
5952  if (Builder.AA->pointsToConstantMemory(PtrVal)) {
5953  Root = Builder.DAG.getEntryNode();
5954  ConstantMemory = true;
5955  } else {
5956  // Do not serialize non-volatile loads against each other.
5957  Root = Builder.DAG.getRoot();
5958  }
5959 
5960  SDValue Ptr = Builder.getValue(PtrVal);
5961  SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
5962  Ptr, MachinePointerInfo(PtrVal),
5963  /* Alignment = */ 1);
5964 
5965  if (!ConstantMemory)
5966  Builder.PendingLoads.push_back(LoadVal.getValue(1));
5967  return LoadVal;
5968 }
5969 
5970 /// processIntegerCallValue - Record the value for an instruction that
5971 /// produces an integer result, converting the type where necessary.
5972 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
5973  SDValue Value,
5974  bool IsSigned) {
5976  I.getType(), true);
5977  if (IsSigned)
5978  Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
5979  else
5980  Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
5981  setValue(&I, Value);
5982 }
5983 
5984 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
5985 /// If so, return true and lower it, otherwise return false and it will be
5986 /// lowered like a normal call.
5987 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
5988  // Verify that the prototype makes sense. int memcmp(void*,void*,size_t)
5989  if (I.getNumArgOperands() != 3)
5990  return false;
5991 
5992  const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
5993  if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
5994  !I.getArgOperand(2)->getType()->isIntegerTy() ||
5995  !I.getType()->isIntegerTy())
5996  return false;
5997 
5998  const Value *Size = I.getArgOperand(2);
5999  const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6000  if (CSize && CSize->getZExtValue() == 0) {
6002  I.getType(), true);
6003  setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6004  return true;
6005  }
6006 
6008  std::pair<SDValue, SDValue> Res =
6010  getValue(LHS), getValue(RHS), getValue(Size),
6011  MachinePointerInfo(LHS),
6012  MachinePointerInfo(RHS));
6013  if (Res.first.getNode()) {
6014  processIntegerCallValue(I, Res.first, true);
6015  PendingLoads.push_back(Res.second);
6016  return true;
6017  }
6018 
6019  // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6020  // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6021  if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
6022  bool ActuallyDoIt = true;
6023  MVT LoadVT;
6024  Type *LoadTy;
6025  switch (CSize->getZExtValue()) {
6026  default:
6027  LoadVT = MVT::Other;
6028  LoadTy = nullptr;
6029  ActuallyDoIt = false;
6030  break;
6031  case 2:
6032  LoadVT = MVT::i16;
6033  LoadTy = Type::getInt16Ty(CSize->getContext());
6034  break;
6035  case 4:
6036  LoadVT = MVT::i32;
6037  LoadTy = Type::getInt32Ty(CSize->getContext());
6038  break;
6039  case 8:
6040  LoadVT = MVT::i64;
6041  LoadTy = Type::getInt64Ty(CSize->getContext());
6042  break;
6043  /*
6044  case 16:
6045  LoadVT = MVT::v4i32;
6046  LoadTy = Type::getInt32Ty(CSize->getContext());
6047  LoadTy = VectorType::get(LoadTy, 4);
6048  break;
6049  */
6050  }
6051 
6052  // This turns into unaligned loads. We only do this if the target natively
6053  // supports the MVT we'll be loading or if it is small enough (<= 4) that
6054  // we'll only produce a small number of byte loads.
6055 
6056  // Require that we can find a legal MVT, and only do this if the target
6057  // supports unaligned loads of that type. Expanding into byte loads would
6058  // bloat the code.
6059  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6060  if (ActuallyDoIt && CSize->getZExtValue() > 4) {
6061  unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6062  unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6063  // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6064  // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6065  // TODO: Check alignment of src and dest ptrs.
6066  if (!TLI.isTypeLegal(LoadVT) ||
6067  !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
6068  !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
6069  ActuallyDoIt = false;
6070  }
6071 
6072  if (ActuallyDoIt) {
6073  SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
6074  SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
6075 
6076  SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
6077  ISD::SETNE);
6078  processIntegerCallValue(I, Res, false);
6079  return true;
6080  }
6081  }
6082 
6083 
6084  return false;
6085 }
6086 
6087 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
6088 /// form. If so, return true and lower it, otherwise return false and it
6089 /// will be lowered like a normal call.
6090 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6091  // Verify that the prototype makes sense. void *memchr(void *, int, size_t)
6092  if (I.getNumArgOperands() != 3)
6093  return false;
6094 
6095  const Value *Src = I.getArgOperand(0);
6096  const Value *Char = I.getArgOperand(1);
6097  const Value *Length = I.getArgOperand(2);
6098  if (!Src->getType()->isPointerTy() ||
6099  !Char->getType()->isIntegerTy() ||
6100  !Length->getType()->isIntegerTy() ||
6101  !I.getType()->isPointerTy())
6102  return false;
6103 
6105  std::pair<SDValue, SDValue> Res =
6107  getValue(Src), getValue(Char), getValue(Length),
6108  MachinePointerInfo(Src));
6109  if (Res.first.getNode()) {
6110  setValue(&I, Res.first);
6111  PendingLoads.push_back(Res.second);
6112  return true;
6113  }
6114 
6115  return false;
6116 }
6117 
6118 ///
6119 /// visitMemPCpyCall -- lower a mempcpy call as a memcpy followed by code to
6120 /// to adjust the dst pointer by the size of the copied memory.
6121 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6122 
6123  // Verify argument count: void *mempcpy(void *, const void *, size_t)
6124  if (I.getNumArgOperands() != 3)
6125  return false;
6126 
6127  SDValue Dst = getValue(I.getArgOperand(0));
6128  SDValue Src = getValue(I.getArgOperand(1));
6129  SDValue Size = getValue(I.getArgOperand(2));
6130 
6131  unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6132  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6133  unsigned Align = std::min(DstAlign, SrcAlign);
6134  if (Align == 0) // Alignment of one or both could not be inferred.
6135  Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6136 
6137  bool isVol = false;
6138  SDLoc sdl = getCurSDLoc();
6139 
6140  // In the mempcpy context we need to pass in a false value for isTailCall
6141  // because the return pointer needs to be adjusted by the size of
6142  // the copied memory.
6143  SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6144  false, /*isTailCall=*/false,
6147  assert(MC.getNode() != nullptr &&
6148  "** memcpy should not be lowered as TailCall in mempcpy context **");
6149  DAG.setRoot(MC);
6150 
6151  // Check if Size needs to be truncated or extended.
6152  Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6153 
6154  // Adjust return pointer to point just past the last dst byte.
6155  SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6156  Dst, Size);
6157  setValue(&I, DstPlusSize);
6158  return true;
6159 }
6160 
6161 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
6162 /// optimized form. If so, return true and lower it, otherwise return false
6163 /// and it will be lowered like a normal call.
6164 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6165  // Verify that the prototype makes sense. char *strcpy(char *, char *)
6166  if (I.getNumArgOperands() != 2)
6167  return false;
6168 
6169  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6170  if (!Arg0->getType()->isPointerTy() ||
6171  !Arg1->getType()->isPointerTy() ||
6172  !I.getType()->isPointerTy())
6173  return false;
6174 
6176  std::pair<SDValue, SDValue> Res =
6178  getValue(Arg0), getValue(Arg1),
6179  MachinePointerInfo(Arg0),
6180  MachinePointerInfo(Arg1), isStpcpy);
6181  if (Res.first.getNode()) {
6182  setValue(&I, Res.first);
6183  DAG.setRoot(Res.second);
6184  return true;
6185  }
6186 
6187  return false;
6188 }
6189 
6190 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
6191 /// If so, return true and lower it, otherwise return false and it will be
6192 /// lowered like a normal call.
6193 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6194  // Verify that the prototype makes sense. int strcmp(void*,void*)
6195  if (I.getNumArgOperands() != 2)
6196  return false;
6197 
6198  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6199  if (!Arg0->getType()->isPointerTy() ||
6200  !Arg1->getType()->isPointerTy() ||
6201  !I.getType()->isIntegerTy())
6202  return false;
6203 
6205  std::pair<SDValue, SDValue> Res =
6207  getValue(Arg0), getValue(Arg1),
6208  MachinePointerInfo(Arg0),
6209  MachinePointerInfo(Arg1));
6210  if (Res.first.getNode()) {
6211  processIntegerCallValue(I, Res.first, true);
6212  PendingLoads.push_back(Res.second);
6213  return true;
6214  }
6215 
6216  return false;
6217 }
6218 
6219 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
6220 /// form. If so, return true and lower it, otherwise return false and it
6221 /// will be lowered like a normal call.
6222 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6223  // Verify that the prototype makes sense. size_t strlen(char *)
6224  if (I.getNumArgOperands() != 1)
6225  return false;
6226 
6227  const Value *Arg0 = I.getArgOperand(0);
6228  if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
6229  return false;
6230 
6232  std::pair<SDValue, SDValue> Res =
6234  getValue(Arg0), MachinePointerInfo(Arg0));
6235  if (Res.first.getNode()) {
6236  processIntegerCallValue(I, Res.first, false);
6237  PendingLoads.push_back(Res.second);
6238  return true;
6239  }
6240 
6241  return false;
6242 }
6243 
6244 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
6245 /// form. If so, return true and lower it, otherwise return false and it
6246 /// will be lowered like a normal call.
6247 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6248  // Verify that the prototype makes sense. size_t strnlen(char *, size_t)
6249  if (I.getNumArgOperands() != 2)
6250  return false;
6251 
6252  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6253  if (!Arg0->getType()->isPointerTy() ||
6254  !Arg1->getType()->isIntegerTy() ||
6255  !I.getType()->isIntegerTy())
6256  return false;
6257 
6259  std::pair<SDValue, SDValue> Res =
6261  getValue(Arg0), getValue(Arg1),
6262  MachinePointerInfo(Arg0));
6263  if (Res.first.getNode()) {
6264  processIntegerCallValue(I, Res.first, false);
6265  PendingLoads.push_back(Res.second);
6266  return true;
6267  }
6268 
6269  return false;
6270 }
6271 
6272 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
6273 /// operation (as expected), translate it to an SDNode with the specified opcode
6274 /// and return true.
6275 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6276  unsigned Opcode) {
6277  // Sanity check that it really is a unary floating-point call.
6278  if (I.getNumArgOperands() != 1 ||
6279  !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
6280  I.getType() != I.getArgOperand(0)->getType() ||
6281  !I.onlyReadsMemory())
6282  return false;
6283 
6284  SDValue Tmp = getValue(I.getArgOperand(0));
6285  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6286  return true;
6287 }
6288 
6289 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
6290 /// operation (as expected), translate it to an SDNode with the specified opcode
6291 /// and return true.
6292 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6293  unsigned Opcode) {
6294  // Sanity check that it really is a binary floating-point call.
6295  if (I.getNumArgOperands() != 2 ||
6296  !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
6297  I.getType() != I.getArgOperand(0)->getType() ||
6298  I.getType() != I.getArgOperand(1)->getType() ||
6299  !I.onlyReadsMemory())
6300  return false;
6301 
6302  SDValue Tmp0 = getValue(I.getArgOperand(0));
6303  SDValue Tmp1 = getValue(I.getArgOperand(1));
6304  EVT VT = Tmp0.getValueType();
6305  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6306  return true;
6307 }
6308 
6309 void SelectionDAGBuilder::visitCall(const CallInst &I) {
6310  // Handle inline assembly differently.
6311  if (isa<InlineAsm>(I.getCalledValue())) {
6312  visitInlineAsm(&I);
6313  return;
6314  }
6315 
6318 
6319  const char *RenameFn = nullptr;
6320  if (Function *F = I.getCalledFunction()) {
6321  if (F->isDeclaration()) {
6322  if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
6323  if (unsigned IID = II->getIntrinsicID(F)) {
6324  RenameFn = visitIntrinsicCall(I, IID);
6325  if (!RenameFn)
6326  return;
6327  }
6328  }
6329  if (Intrinsic::ID IID = F->getIntrinsicID()) {
6330  RenameFn = visitIntrinsicCall(I, IID);
6331  if (!RenameFn)
6332  return;
6333  }
6334  }
6335 
6336  // Check for well-known libc/libm calls. If the function is internal, it
6337  // can't be a library call. Don't do the check if marked as nobuiltin for
6338  // some reason.
6340  if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
6341  LibInfo->getLibFunc(F->getName(), Func) &&
6342  LibInfo->hasOptimizedCodeGen(Func)) {
6343  switch (Func) {
6344  default: break;
6345  case LibFunc::copysign:
6346  case LibFunc::copysignf:
6347  case LibFunc::copysignl:
6348  if (I.getNumArgOperands() == 2 && // Basic sanity checks.
6350  I.getType() == I.getArgOperand(0)->getType() &&
6351  I.getType() == I.getArgOperand(1)->getType() &&
6352  I.onlyReadsMemory()) {
6353  SDValue LHS = getValue(I.getArgOperand(0));
6354  SDValue RHS = getValue(I.getArgOperand(1));
6356  LHS.getValueType(), LHS, RHS));
6357  return;
6358  }
6359  break;
6360  case LibFunc::fabs:
6361  case LibFunc::fabsf:
6362  case LibFunc::fabsl:
6363  if (visitUnaryFloatCall(I, ISD::FABS))
6364  return;
6365  break;
6366  case LibFunc::fmin:
6367  case LibFunc::fminf:
6368  case LibFunc::fminl:
6369  if (visitBinaryFloatCall(I, ISD::FMINNUM))
6370  return;
6371  break;
6372  case LibFunc::fmax:
6373  case LibFunc::fmaxf:
6374  case LibFunc::fmaxl:
6375  if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6376  return;
6377  break;
6378  case LibFunc::sin:
6379  case LibFunc::sinf:
6380  case LibFunc::sinl:
6381  if (visitUnaryFloatCall(I, ISD::FSIN))
6382  return;
6383  break;
6384  case LibFunc::cos:
6385  case LibFunc::cosf:
6386  case LibFunc::cosl:
6387  if (visitUnaryFloatCall(I, ISD::FCOS))
6388  return;
6389  break;
6390  case LibFunc::sqrt:
6391  case LibFunc::sqrtf:
6392  case LibFunc::sqrtl:
6393  case LibFunc::sqrt_finite:
6394  case LibFunc::sqrtf_finite:
6395  case LibFunc::sqrtl_finite:
6396  if (visitUnaryFloatCall(I, ISD::FSQRT))
6397  return;
6398  break;
6399  case LibFunc::floor:
6400  case LibFunc::floorf:
6401  case LibFunc::floorl:
6402  if (visitUnaryFloatCall(I, ISD::FFLOOR))
6403  return;
6404  break;
6405  case LibFunc::nearbyint:
6406  case LibFunc::nearbyintf:
6407  case LibFunc::nearbyintl:
6408  if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6409  return;
6410  break;
6411  case LibFunc::ceil:
6412  case LibFunc::ceilf:
6413  case LibFunc::ceill:
6414  if (visitUnaryFloatCall(I, ISD::FCEIL))
6415  return;
6416  break;
6417  case LibFunc::rint:
6418  case LibFunc::rintf:
6419  case LibFunc::rintl:
6420  if (visitUnaryFloatCall(I, ISD::FRINT))
6421  return;
6422  break;
6423  case LibFunc::round:
6424  case LibFunc::roundf:
6425  case LibFunc::roundl:
6426  if (visitUnaryFloatCall(I, ISD::FROUND))
6427  return;
6428  break;
6429  case LibFunc::trunc:
6430  case LibFunc::truncf:
6431  case LibFunc::truncl:
6432  if (visitUnaryFloatCall(I, ISD::FTRUNC))
6433  return;
6434  break;
6435  case LibFunc::log2:
6436  case LibFunc::log2f:
6437  case LibFunc::log2l:
6438  if (visitUnaryFloatCall(I, ISD::FLOG2))
6439  return;
6440  break;
6441  case LibFunc::exp2:
6442  case LibFunc::exp2f:
6443  case LibFunc::exp2l:
6444  if (visitUnaryFloatCall(I, ISD::FEXP2))
6445  return;
6446  break;
6447  case LibFunc::memcmp:
6448  if (visitMemCmpCall(I))
6449  return;
6450  break;
6451  case LibFunc::mempcpy:
6452  if (visitMemPCpyCall(I))
6453  return;
6454  break;
6455  case LibFunc::memchr:
6456  if (visitMemChrCall(I))
6457  return;
6458  break;
6459  case LibFunc::strcpy:
6460  if (visitStrCpyCall(I, false))
6461  return;
6462  break;
6463  case LibFunc::stpcpy:
6464  if (visitStrCpyCall(I, true))
6465  return;
6466  break;
6467  case LibFunc::strcmp:
6468  if (visitStrCmpCall(I))
6469  return;
6470  break;
6471  case LibFunc::strlen:
6472  if (visitStrLenCall(I))
6473  return;
6474  break;
6475  case LibFunc::strnlen:
6476  if (visitStrNLenCall(I))
6477  return;
6478  break;
6479  }
6480  }
6481  }
6482 
6483  SDValue Callee;
6484  if (!RenameFn)
6485  Callee = getValue(I.getCalledValue());
6486  else
6487  Callee = DAG.getExternalSymbol(
6488  RenameFn,
6490 
6491  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
6492  // have to do anything here to lower funclet bundles.
6495  "Cannot lower calls with arbitrary operand bundles!");
6496 
6498  LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
6499  else
6500  // Check if we can potentially perform a tail call. More detailed checking
6501  // is be done within LowerCallTo, after more information about the call is
6502  // known.
6503  LowerCallTo(&I, Callee, I.isTailCall());
6504 }
6505 
6506 namespace {
6507 
6508 /// AsmOperandInfo - This contains information for each constraint that we are
6509 /// lowering.
6510 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
6511 public:
6512  /// CallOperand - If this is the result output operand or a clobber
6513  /// this is null, otherwise it is the incoming operand to the CallInst.
6514  /// This gets modified as the asm is processed.
6515  SDValue CallOperand;
6516 
6517  /// AssignedRegs - If this is a register or register class operand, this
6518  /// contains the set of register corresponding to the operand.
6519  RegsForValue AssignedRegs;
6520 
6521  explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
6522  : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
6523  }
6524 
6525  /// Whether or not this operand accesses memory
6526  bool hasMemory(const TargetLowering &TLI) const {
6527  // Indirect operand accesses access memory.
6528  if (isIndirect)
6529  return true;
6530 
6531  for (const auto &Code : Codes)
6533  return true;
6534 
6535  return false;
6536  }
6537 
6538  /// getCallOperandValEVT - Return the EVT of the Value* that this operand
6539  /// corresponds to. If there is no Value* for this operand, it returns
6540  /// MVT::Other.
6541  EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
6542  const DataLayout &DL) const {
6543  if (!CallOperandVal) return MVT::Other;
6544 
6545  if (isa<BasicBlock>(CallOperandVal))
6546  return TLI.getPointerTy(DL);
6547 
6548  llvm::Type *OpTy = CallOperandVal->getType();
6549 
6550  // FIXME: code duplicated from TargetLowering::ParseConstraints().
6551  // If this is an indirect operand, the operand is a pointer to the
6552  // accessed type.
6553  if (isIndirect) {
6554  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
6555  if (!PtrTy)
6556  report_fatal_error("Indirect operand for inline asm not a pointer!");
6557  OpTy = PtrTy->getElementType();
6558  }
6559 
6560  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6561  if (StructType *STy = dyn_cast<StructType>(OpTy))
6562  if (STy->getNumElements() == 1)
6563  OpTy = STy->getElementType(0);
6564 
6565  // If OpTy is not a single value, it may be a struct/union that we
6566  // can tile with integers.
6567  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6568  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6569  switch (BitSize) {
6570  default: break;
6571  case 1:
6572  case 8:
6573  case 16:
6574  case 32:
6575  case 64:
6576  case 128:
6577  OpTy = IntegerType::get(Context, BitSize);
6578  break;
6579  }
6580  }
6581 
6582  return TLI.getValueType(DL, OpTy, true);
6583  }
6584 };
6585 
6586 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
6587 
6588 } // end anonymous namespace
6589 
6590 /// Make sure that the output operand \p OpInfo and its corresponding input
6591 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
6592 /// out).
6593 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
6594  SDISelAsmOperandInfo &MatchingOpInfo,
6595  SelectionDAG &DAG) {
6596  if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
6597  return;
6598 
6599  const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
6600  const auto &TLI = DAG.getTargetLoweringInfo();
6601 
6602  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6603  TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6604  OpInfo.ConstraintVT);
6605  std::pair<unsigned, const TargetRegisterClass *> InputRC =
6606  TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
6607  MatchingOpInfo.ConstraintVT);
6608  if ((OpInfo.ConstraintVT.isInteger() !=
6609  MatchingOpInfo.ConstraintVT.isInteger()) ||
6610  (MatchRC.second != InputRC.second)) {
6611  // FIXME: error out in a more elegant fashion
6612  report_fatal_error("Unsupported asm: input constraint"
6613  " with a matching output constraint of"
6614  " incompatible type!");
6615  }
6616  MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
6617 }
6618 
6619 /// Get a direct memory input to behave well as an indirect operand.
6620 /// This may introduce stores, hence the need for a \p Chain.
6621 /// \return The (possibly updated) chain.
6622 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
6623  SDISelAsmOperandInfo &OpInfo,
6624  SelectionDAG &DAG) {
6625  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6626 
6627  // If we don't have an indirect input, put it in the constpool if we can,
6628  // otherwise spill it to a stack slot.
6629  // TODO: This isn't quite right. We need to handle these according to
6630  // the addressing mode that the constraint wants. Also, this may take
6631  // an additional register for the computation and we don't want that
6632  // either.
6633 
6634  // If the operand is a float, integer, or vector constant, spill to a
6635  // constant pool entry to get its address.
6636  const Value *OpVal = OpInfo.CallOperandVal;
6637  if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6638  isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6639  OpInfo.CallOperand = DAG.getConstantPool(
6640  cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6641  return Chain;
6642  }
6643 
6644  // Otherwise, create a stack slot and emit a store to it before the asm.
6645  Type *Ty = OpVal->getType();
6646  auto &DL = DAG.getDataLayout();
6647  uint64_t TySize = DL.getTypeAllocSize(Ty);
6648  unsigned Align = DL.getPrefTypeAlignment(Ty);
6649  MachineFunction &MF = DAG.getMachineFunction();
6650  int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
6651  SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy(DL));
6652  Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
6654  OpInfo.CallOperand = StackSlot;
6655 
6656  return Chain;
6657 }
6658 
6659 /// GetRegistersForValue - Assign registers (virtual or physical) for the
6660 /// specified operand. We prefer to assign virtual registers, to allow the
6661 /// register allocator to handle the assignment process. However, if the asm
6662 /// uses features that we can't model on machineinstrs, we have SDISel do the
6663 /// allocation. This produces generally horrible, but correct, code.
6664 ///
6665 /// OpInfo describes the operand.
6666 ///
6667 static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
6668  const SDLoc &DL,
6669  SDISelAsmOperandInfo &OpInfo) {
6670  LLVMContext &Context = *DAG.getContext();
6671 
6672  MachineFunction &MF = DAG.getMachineFunction();
6674 
6675  // If this is a constraint for a single physreg, or a constraint for a
6676  // register class, find it.
6677  std::pair<unsigned, const TargetRegisterClass *> PhysReg =
6679  OpInfo.ConstraintCode,
6680  OpInfo.ConstraintVT);
6681 
6682  unsigned NumRegs = 1;
6683  if (OpInfo.ConstraintVT != MVT::Other) {
6684  // If this is a FP input in an integer register (or visa versa) insert a bit
6685  // cast of the input value. More generally, handle any case where the input
6686  // value disagrees with the register class we plan to stick this in.
6687  if (OpInfo.Type == InlineAsm::isInput &&
6688  PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
6689  // Try to convert to the first EVT that the reg class contains. If the
6690  // types are identical size, use a bitcast to convert (e.g. two differing
6691  // vector types).
6692  MVT RegVT = *PhysReg.second->vt_begin();
6693  if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
6694  OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6695  RegVT, OpInfo.CallOperand);
6696  OpInfo.ConstraintVT = RegVT;
6697  } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
6698  // If the input is a FP value and we want it in FP registers, do a
6699  // bitcast to the corresponding integer type. This turns an f64 value
6700  // into i64, which can be passed with two i32 values on a 32-bit
6701  // machine.
6702  RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
6703  OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6704  RegVT, OpInfo.CallOperand);
6705  OpInfo.ConstraintVT = RegVT;
6706  }
6707  }
6708 
6709  NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
6710  }
6711 
6712  MVT RegVT;
6713  EVT ValueVT = OpInfo.ConstraintVT;
6714 
6715  // If this is a constraint for a specific physical register, like {r17},
6716  // assign it now.
6717  if (unsigned AssignedReg = PhysReg.first) {
6718  const TargetRegisterClass *RC = PhysReg.second;
6719  if (OpInfo.ConstraintVT == MVT::Other)
6720  ValueVT = *RC->vt_begin();
6721 
6722  // Get the actual register value type. This is important, because the user
6723  // may have asked for (e.g.) the AX register in i32 type. We need to
6724  // remember that AX is actually i16 to get the right extension.
6725  RegVT = *RC->vt_begin();
6726 
6727  // This is a explicit reference to a physical register.
6728  Regs.push_back(AssignedReg);
6729 
6730  // If this is an expanded reference, add the rest of the regs to Regs.
6731  if (NumRegs != 1) {
6733  for (; *I != AssignedReg; ++I)
6734  assert(I != RC->end() && "Didn't find reg!");
6735 
6736  // Already added the first reg.
6737  --NumRegs; ++I;
6738  for (; NumRegs; --NumRegs, ++I) {
6739  assert(I != RC->end() && "Ran out of registers to allocate!");
6740  Regs.push_back(*I);
6741  }
6742  }
6743 
6744  OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6745  return;
6746  }
6747 
6748  // Otherwise, if this was a reference to an LLVM register class, create vregs
6749  // for this reference.
6750  if (const TargetRegisterClass *RC = PhysReg.second) {
6751  RegVT = *RC->vt_begin();
6752  if (OpInfo.ConstraintVT == MVT::Other)
6753  ValueVT = RegVT;
6754 
6755  // Create the appropriate number of virtual registers.
6756  MachineRegisterInfo &RegInfo = MF.getRegInfo();
6757  for (; NumRegs; --NumRegs)
6758  Regs.push_back(RegInfo.createVirtualRegister(RC));
6759 
6760  OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6761  return;
6762  }
6763 
6764  // Otherwise, we couldn't allocate enough registers for this.
6765 }
6766 
6767 static unsigned
6768 findMatchingInlineAsmOperand(unsigned OperandNo,
6769  const std::vector<SDValue> &AsmNodeOperands) {
6770  // Scan until we find the definition we already emitted of this operand.
6771  unsigned CurOp = InlineAsm::Op_FirstOperand;
6772  for (; OperandNo; --OperandNo) {
6773  // Advance to the next operand.
6774  unsigned OpFlag =
6775  cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6776  assert((InlineAsm::isRegDefKind(OpFlag) ||
6778  InlineAsm::isMemKind(OpFlag)) &&
6779  "Skipped past definitions?");
6780  CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
6781  }
6782  return CurOp;
6783 }
6784 
6785 /// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
6786 /// \return true if it has succeeded, false otherwise
6787 static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
6788  MVT RegVT, SelectionDAG &DAG) {
6789  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6791  for (unsigned i = 0, e = NumRegs; i != e; ++i) {
6792  if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
6793  Regs.push_back(RegInfo.createVirtualRegister(RC));
6794  else
6795  return false;
6796  }
6797  return true;
6798 }
6799 
6800 class ExtraFlags {
6801  unsigned Flags = 0;
6802 
6803 public:
6805  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6806  if (IA->hasSideEffects())
6808  if (IA->isAlignStack())
6810  if (CS.isConvergent())
6812  Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
6813  }
6814 
6816  // Ideally, we would only check against memory constraints. However, the
6817  // meaning of an Other constraint can be target-specific and we can't easily
6818  // reason about it. Therefore, be conservative and set MayLoad/MayStore
6819  // for Other constraints as well.
6820  if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
6822  if (OpInfo.Type == InlineAsm::isInput)
6823  Flags |= InlineAsm::Extra_MayLoad;
6824  else if (OpInfo.Type == InlineAsm::isOutput)
6825  Flags |= InlineAsm::Extra_MayStore;
6826  else if (OpInfo.Type == InlineAsm::isClobber)
6828  }
6829  }
6830 
6831  unsigned get() const { return Flags; }
6832 };
6833 
6834 /// visitInlineAsm - Handle a call to an InlineAsm object.
6835 ///
6836 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
6837  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6838 
6839  /// ConstraintOperands - Information about all of the constraints.
6840  SDISelAsmOperandInfoVector ConstraintOperands;
6841 
6842  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6845 
6846  bool hasMemory = false;
6847 
6848  // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6849  ExtraFlags ExtraInfo(CS);
6850 
6851  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6852  unsigned ResNo = 0; // ResNo - The result number of the next output.
6853  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
6854  ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
6855  SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
6856 
6857  MVT OpVT = MVT::Other;
6858 
6859  // Compute the value type for each operand.
6860  if (OpInfo.Type == InlineAsm::isInput ||
6861  (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
6862  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
6863 
6864  // Process the call argument. BasicBlocks are labels, currently appearing
6865  // only in asm's.
6866  if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
6867  OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
6868  } else {
6869  OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
6870  }
6871 
6872  OpVT =
6873  OpInfo
6874  .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
6875  .getSimpleVT();
6876  }
6877 
6878  if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
6879  // The return value of the call is this value. As such, there is no
6880  // corresponding argument.
6881  assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
6882  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
6883  OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
6884  STy->getElementType(ResNo));
6885  } else {
6886  assert(ResNo == 0 && "Asm only has one result!");
6887  OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
6888  }
6889  ++ResNo;
6890  }
6891 
6892  OpInfo.ConstraintVT = OpVT;
6893 
6894  if (!hasMemory)
6895  hasMemory = OpInfo.hasMemory(TLI);
6896 
6897  // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
6898  // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
6899  auto TargetConstraint = TargetConstraints[i];
6900 
6901  // Compute the constraint code and ConstraintType to use.
6902  TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
6903 
6904  ExtraInfo.update(TargetConstraint);
6905  }
6906 
6907  SDValue Chain, Flag;
6908 
6909  // We won't need to flush pending loads if this asm doesn't touch
6910  // memory and is nonvolatile.
6911  if (hasMemory || IA->hasSideEffects())
6912  Chain = getRoot();
6913  else
6914  Chain = DAG.getRoot();
6915 
6916  // Second pass over the constraints: compute which constraint option to use
6917  // and assign registers to constraints that want a specific physreg.
6918  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6919  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6920 
6921  // If this is an output operand with a matching input operand, look up the
6922  // matching input. If their types mismatch, e.g. one is an integer, the
6923  // other is floating point, or their sizes are different, flag it as an
6924  // error.
6925  if (OpInfo.hasMatchingInput()) {
6926  SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6927  patchMatchingInput(OpInfo, Input, DAG);
6928  }
6929 
6930  // Compute the constraint code and ConstraintType to use.
6931  TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
6932 
6933  if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6934  OpInfo.Type == InlineAsm::isClobber)
6935  continue;
6936 
6937  // If this is a memory input, and if the operand is not indirect, do what we
6938  // need to to provide an address for the memory input.
6939  if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6940  !OpInfo.isIndirect) {
6941  assert((OpInfo.isMultipleAlternative ||
6942  (OpInfo.Type == InlineAsm::isInput)) &&
6943  "Can only indirectify direct input operands!");
6944 
6945  // Memory operands really want the address of the value.
6946  Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
6947 
6948  // There is no longer a Value* corresponding to this operand.
6949  OpInfo.CallOperandVal = nullptr;
6950 
6951  // It is now an indirect operand.
6952  OpInfo.isIndirect = true;
6953  }
6954 
6955  // If this constraint is for a specific register, allocate it before
6956  // anything else.
6957  if (OpInfo.ConstraintType == TargetLowering::C_Register)
6958  GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6959  }
6960 
6961  // Third pass - Loop over all of the operands, assigning virtual or physregs
6962  // to register class operands.
6963  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6964  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6965 
6966  // C_Register operands have already been allocated, Other/Memory don't need
6967  // to be.
6968  if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
6969  GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6970  }
6971 
6972  // AsmNodeOperands - The operands for the ISD::INLINEASM node.
6973  std::vector<SDValue> AsmNodeOperands;
6974  AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
6975  AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
6976  IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
6977 
6978  // If we have a !srcloc metadata node associated with it, we want to attach
6979  // this to the ultimately generated inline asm machineinstr. To do this, we
6980  // pass in the third operand as this (potentially null) inline asm MDNode.
6981  const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
6982  AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
6983 
6984  // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6985  // bits as operand 3.
6986  AsmNodeOperands.push_back(DAG.getTargetConstant(
6987  ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6988 
6989  // Loop over all of the inputs, copying the operand values into the
6990  // appropriate registers and processing the output regs.
6991  RegsForValue RetValRegs;
6992 
6993  // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
6994  std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
6995 
6996  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6997  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6998 
6999  switch (OpInfo.Type) {
7000  case InlineAsm::isOutput: {
7001  if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7002  OpInfo.ConstraintType != TargetLowering::C_Register) {
7003  // Memory output, or 'other' output (e.g. 'X' constraint).
7004  assert(OpInfo.isIndirect && "Memory output must be indirect operand");
7005 
7006  unsigned ConstraintID =
7007  TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7008  assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7009  "Failed to convert memory constraint code to constraint id.");
7010 
7011  // Add information to the INLINEASM node to know about this output.
7012  unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7013  OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7014  AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7015  MVT::i32));
7016  AsmNodeOperands.push_back(OpInfo.CallOperand);
7017  break;
7018  }
7019 
7020  // Otherwise, this is a register or register class output.
7021 
7022  // Copy the output from the appropriate register. Find a register that
7023  // we can use.
7024  if (OpInfo.AssignedRegs.Regs.empty()) {
7025  emitInlineAsmError(
7026  CS, "couldn't allocate output register for constraint '" +
7027  Twine(OpInfo.ConstraintCode) + "'");
7028  return;
7029  }
7030 
7031  // If this is an indirect operand, store through the pointer after the
7032  // asm.
7033  if (OpInfo.isIndirect) {
7034  IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7035  OpInfo.CallOperandVal));
7036  } else {
7037  // This is the result value of the call.
7038  assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7039  // Concatenate this output onto the outputs list.
7040  RetValRegs.append(OpInfo.AssignedRegs);
7041  }
7042 
7043  // Add information to the INLINEASM node to know that this register is
7044  // set.
7045  OpInfo.AssignedRegs
7046  .AddInlineAsmOperands(OpInfo.isEarlyClobber
7049  false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7050  break;
7051  }
7052  case InlineAsm::isInput: {
7053  SDValue InOperandVal = OpInfo.CallOperand;
7054 
7055  if (OpInfo.isMatchingInputConstraint()) {
7056  // If this is required to match an output register we have already set,
7057  // just use its register.
7058  auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7059  AsmNodeOperands);
7060  unsigned OpFlag =
7061  cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7062  if (InlineAsm::isRegDefKind(OpFlag) ||
7064  // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7065  if (OpInfo.isIndirect) {
7066  // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7067  emitInlineAsmError(CS, "inline asm not supported yet:"
7068  " don't know how to handle tied "
7069  "indirect register inputs");
7070  return;
7071  }
7072 
7073  MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7075 
7076  if (!createVirtualRegs(Regs,
7078  RegVT, DAG)) {
7079  emitInlineAsmError(CS, "inline asm error: This value type register "
7080  "class is not natively supported!");
7081  return;
7082  }
7083 
7084  RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7085 
7086  SDLoc dl = getCurSDLoc();
7087  // Use the produced MatchedRegs object to
7088  MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7089  Chain, &Flag, CS.getInstruction());
7090  MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7091  true, OpInfo.getMatchedOperand(), dl,
7092  DAG, AsmNodeOperands);
7093  break;
7094  }
7095 
7096  assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
7098  "Unexpected number of operands");
7099  // Add information to the INLINEASM node to know about this input.
7100  // See InlineAsm.h isUseOperandTiedToDef.
7102  OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7103  OpInfo.getMatchedOperand());
7104  AsmNodeOperands.push_back(DAG.getTargetConstant(
7105  OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7106  AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7107  break;
7108  }
7109 
7110  // Treat indirect 'X' constraint as memory.
7111  if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7112  OpInfo.isIndirect)
7113  OpInfo.ConstraintType = TargetLowering::C_Memory;
7114 
7115  if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7116  std::vector<SDValue> Ops;
7117  TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7118  Ops, DAG);
7119  if (Ops.empty()) {
7120  emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7121  Twine(OpInfo.ConstraintCode) + "'");
7122  return;
7123  }
7124 
7125  // Add information to the INLINEASM node to know about this input.
7126  unsigned ResOpType =
7128  AsmNodeOperands.push_back(DAG.getTargetConstant(
7129  ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7130  AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7131  break;
7132  }
7133 
7134  if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7135  assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
7136  assert(InOperandVal.getValueType() ==
7137  TLI.getPointerTy(DAG.getDataLayout()) &&
7138  "Memory operands expect pointer values");
7139 
7140  unsigned ConstraintID =
7141  TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7142  assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7143  "Failed to convert memory constraint code to constraint id.");
7144 
7145  // Add information to the INLINEASM node to know about this input.
7146  unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7147  ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7148  AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7149  getCurSDLoc(),
7150  MVT::i32));
7151  AsmNodeOperands.push_back(InOperandVal);
7152  break;
7153  }
7154 
7155  assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
7156  OpInfo.ConstraintType == TargetLowering::C_Register) &&
7157  "Unknown constraint type!");
7158 
7159  // TODO: Support this.
7160  if (OpInfo.isIndirect) {
7161  emitInlineAsmError(
7162  CS, "Don't know how to handle indirect register inputs yet "
7163  "for constraint '" +
7164  Twine(OpInfo.ConstraintCode) + "'");
7165  return;
7166  }
7167 
7168  // Copy the input into the appropriate registers.
7169  if (OpInfo.AssignedRegs.Regs.empty()) {
7170  emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7171  Twine(OpInfo.ConstraintCode) + "'");
7172  return;
7173  }
7174 
7175  SDLoc dl = getCurSDLoc();
7176 
7177  OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7178  Chain, &Flag, CS.getInstruction());
7179 
7180  OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7181  dl, DAG, AsmNodeOperands);
7182  break;
7183  }
7184  case InlineAsm::isClobber: {
7185  // Add the clobbered value to the operand list, so that the register
7186  // allocator is aware that the physreg got clobbered.
7187  if (!OpInfo.AssignedRegs.Regs.empty())
7188  OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7189  false, 0, getCurSDLoc(), DAG,
7190  AsmNodeOperands);
7191  break;
7192  }
7193  }
7194  }
7195 
7196  // Finish up input operands. Set the input chain and add the flag last.
7197  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7198  if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7199 
7201  DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7202  Flag = Chain.getValue(1);
7203 
7204  // If this asm returns a register value, copy the result from that register
7205  // and set it as the value of the call.
7206  if (!RetValRegs.Regs.empty()) {
7207  SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7208  Chain, &Flag, CS.getInstruction());
7209 
7210  // FIXME: Why don't we do this for inline asms with MRVs?
7211  if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7212  EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7213 
7214  // If any of the results of the inline asm is a vector, it may have the
7215  // wrong width/num elts. This can happen for register classes that can
7216  // contain multiple different value types. The preg or vreg allocated may
7217  // not have the same VT as was expected. Convert it to the right type
7218  // with bit_convert.
7219  if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
7221  ResultType, Val);
7222 
7223  } else if (ResultType != Val.getValueType() &&
7224  ResultType.isInteger() && Val.getValueType().isInteger()) {
7225  // If a result value was tied to an input value, the computed result may
7226  // have a wider width than the expected result. Extract the relevant
7227  // portion.
7228  Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7229  }
7230 
7231  assert(ResultType == Val.getValueType() && "Asm result value mismatch!");
7232  }
7233 
7234  setValue(CS.getInstruction(), Val);
7235  // Don't need to use this as a chain in this case.
7236  if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7237  return;
7238  }
7239 
7240  std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
7241 
7242  // Process indirect outputs, first output all of the flagged copies out of
7243  // physregs.
7244  for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7245  RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7246  const Value *Ptr = IndirectStoresToEmit[i].second;
7247  SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7248  Chain, &Flag, IA);
7249  StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7250  }
7251 
7252  // Emit the non-flagged stores from the physregs.
7253  SmallVector<SDValue, 8> OutChains;
7254  for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7255  SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7256  getValue(StoresToEmit[i].second),
7257  MachinePointerInfo(StoresToEmit[i].second));
7258  OutChains.push_back(Val);
7259  }
7260 
7261  if (!OutChains.empty())
7262  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7263 
7264  DAG.setRoot(Chain);
7265 }
7266 
7267 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7268  const Twine &Message) {
7269  LLVMContext &Ctx = *DAG.getContext();
7270  Ctx.emitError(CS.getInstruction(), Message);
7271 
7272  // Make sure we leave the DAG in a valid state
7273  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7274  auto VT = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7275  setValue(CS.getInstruction(), DAG.getUNDEF(VT));
7276 }
7277 
7278 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7280  MVT::Other, getRoot(),
7281  getValue(I.getArgOperand(0)),
7282  DAG.getSrcValue(I.getArgOperand(0))));
7283 }
7284 
7285 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7286  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7287  const DataLayout &DL = DAG.getDataLayout();
7289  getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7290  DAG.getSrcValue(I.getOperand(0)),
7291  DL.getABITypeAlignment(I.getType()));
7292  setValue(&I, V);
7293  DAG.setRoot(V.getValue(1));
7294 }
7295 
7296 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7298  MVT::Other, getRoot(),
7299  getValue(I.getArgOperand(0)),
7300  DAG.getSrcValue(I.getArgOperand(0))));
7301 }
7302 
7303 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7305  MVT::Other, getRoot(),
7306  getValue(I.getArgOperand(0)),
7307  getValue(I.getArgOperand(1)),
7309  DAG.getSrcValue(I.getArgOperand(1))));
7310 }
7311 
7313  const Instruction &I,
7314  SDValue Op) {
7315  const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7316  if (!Range)
7317  return Op;
7318 
7320  if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7321  return Op;
7322 
7323  APInt Lo = CR.getUnsignedMin();
7324  if (!Lo.isMinValue())
7325  return Op;
7326 
7327  APInt Hi = CR.getUnsignedMax();
7328  unsigned Bits = Hi.getActiveBits();
7329 
7330  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7331 
7332  SDLoc SL = getCurSDLoc();
7333 
7334  SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7335  DAG.getValueType(SmallVT));
7336  unsigned NumVals = Op.getNode()->getNumValues();
7337  if (NumVals == 1)
7338  return ZExt;
7339 
7341 
7342  Ops.push_back(ZExt);
7343  for (unsigned I = 1; I != NumVals; ++I)
7344  Ops.push_back(Op.getValue(I));
7345 
7346  return DAG.getMergeValues(Ops, SL);
7347 }
7348 
7349 /// \brief Populate a CallLowerinInfo (into \p CLI) based on the properties of
7350 /// the call being lowered.
7351 ///
7352 /// This is a helper for lowering intrinsics that follow a target calling
7353 /// convention or require stack pointer adjustment. Only a subset of the
7354 /// intrinsic's operands need to participate in the calling convention.
7357  unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7358  bool IsPatchPoint) {
7360  Args.reserve(NumArgs);
7361 
7362  // Populate the argument list.
7363  // Attributes for args start at offset 1, after the return attribute.
7364  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1;
7365  ArgI != ArgE; ++ArgI) {
7366  const Value *V = CS->getOperand(ArgI);
7367 
7368  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
7369 
7371  Entry.Node = getValue(V);
7372  Entry.Ty = V->getType();
7373  Entry.setAttributes(&CS, AttrI);
7374  Args.push_back(Entry);
7375  }
7376 
7377  CLI.setDebugLoc(getCurSDLoc())
7378  .setChain(getRoot())
7379  .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7380  .setDiscardResult(CS->use_empty())
7381  .setIsPatchPoint(IsPatchPoint);
7382 }
7383 
7384 /// \brief Add a stack map intrinsic call's live variable operands to a stackmap
7385 /// or patchpoint target node's operand list.
7386 ///
7387 /// Constants are converted to TargetConstants purely as an optimization to
7388 /// avoid constant materialization and register allocation.
7389 ///
7390 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7391 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
7392 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7393 /// address materialization and register allocation, but may also be required
7394 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7395 /// alloca in the entry block, then the runtime may assume that the alloca's
7396 /// StackMap location can be read immediately after compilation and that the
7397 /// location is valid at any point during execution (this is similar to the
7398 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7399 /// only available in a register, then the runtime would need to trap when
7400 /// execution reaches the StackMap in order to read the alloca's location.
7401 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7402  const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7403  SelectionDAGBuilder &Builder) {
7404  for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7405  SDValue OpVal = Builder.getValue(CS.getArgument(i));
7406  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7407  Ops.push_back(
7409  Ops.push_back(
7410  Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7411  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7412  const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7413  Ops.push_back(Builder.DAG.getTargetFrameIndex(
7414  FI->getIndex(), TLI.getPointerTy(Builder.DAG.getDataLayout())));
7415  } else
7416  Ops.push_back(OpVal);
7417  }
7418 }
7419 
7420 /// \brief Lower llvm.experimental.stackmap directly to its target opcode.
7421 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7422  // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7423  // [live variables...])
7424 
7425  assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
7426 
7427  SDValue Chain, InFlag, Callee, NullPtr;
7429 
7430  SDLoc DL = getCurSDLoc();
7431  Callee = getValue(CI.getCalledValue());
7432  NullPtr = DAG.getIntPtrConstant(0, DL, true);
7433 
7434  // The stackmap intrinsic only records the live variables (the arguemnts
7435  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7436  // intrinsic, this won't be lowered to a function call. This means we don't
7437  // have to worry about calling conventions and target specific lowering code.
7438  // Instead we perform the call lowering right here.
7439  //
7440  // chain, flag = CALLSEQ_START(chain, 0)
7441  // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
7442  // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
7443  //
7444  Chain = DAG.getCALLSEQ_START(getRoot(), NullPtr, DL);
7445  InFlag = Chain.getValue(1);
7446 
7447  // Add the <id> and <numBytes> constants.
7450  cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
7453  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
7454  MVT::i32));
7455 
7456  // Push live variables for the stack map.
7457  addStackMapLiveVars(&CI, 2, DL, Ops, *this);
7458 
7459  // We are not pushing any register mask info here on the operands list,
7460  // because the stackmap doesn't clobber anything.
7461 
7462  // Push the chain and the glue flag.
7463  Ops.push_back(Chain);
7464  Ops.push_back(InFlag);
7465 
7466  // Create the STACKMAP node.
7468  SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
7469  Chain = SDValue(SM, 0);
7470  InFlag = Chain.getValue(1);
7471 
7472  Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
7473 
7474  // Stackmaps don't generate values, so nothing goes into the NodeMap.
7475 
7476  // Set the root to the target-lowered call chain.
7477  DAG.setRoot(Chain);
7478 
7479  // Inform the Frame Information that we have a stackmap in this function.
7481 }
7482 
7483 /// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
7484 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
7485  const BasicBlock *EHPadBB) {
7486  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
7487  // i32 <numBytes>,
7488  // i8* <target>,
7489  // i32 <numArgs>,
7490  // [Args...],
7491  // [live variables...])
7492 
7493  CallingConv::ID CC = CS.getCallingConv();
7494  bool IsAnyRegCC = CC == CallingConv::AnyReg;
7495  bool HasDef = !CS->getType()->isVoidTy();
7496  SDLoc dl = getCurSDLoc();
7497  SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
7498 
7499  // Handle immediate and symbolic callees.
7500  if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
7501  Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
7502  /*isTarget=*/true);
7503  else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
7504  Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
7505  SDLoc(SymbolicCallee),
7506  SymbolicCallee->getValueType(0));
7507 
7508  // Get the real number of arguments participating in the call <numArgs>
7510  unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
7511 
7512  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
7513  // Intrinsics include all meta-operands up to but not including CC.
7514  unsigned NumMetaOpers = PatchPointOpers::CCPos;
7515  assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
7516  "Not enough arguments provided to the patchpoint intrinsic");
7517 
7518  // For AnyRegCC the arguments are lowered later on manually.
7519  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
7520  Type *ReturnTy =
7521  IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
7522 
7524  populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
7525  true);
7526  std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7527 
7528  SDNode *CallEnd = Result.second.getNode();
7529  if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
7530  CallEnd = CallEnd->getOperand(0).getNode();
7531 
7532  /// Get a call instruction from the call sequence chain.
7533  /// Tail calls are not allowed.
7534  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
7535  "Expected a callseq node.");
7536  SDNode *Call = CallEnd->getOperand(0).getNode();
7537  bool HasGlue = Call->getGluedNode();
7538 
7539  // Replace the target specific call node with the patchable intrinsic.
7541 
7542  // Add the <id> and <numBytes> constants.
7543  SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
7545  cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
7546  SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
7548  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
7549  MVT::i32));
7550 
7551  // Add the callee.
7552  Ops.push_back(Callee);
7553 
7554  // Adjust <numArgs> to account for any arguments that have been passed on the
7555  // stack instead.
7556  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
7557  unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
7558  NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
7559  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
7560 
7561  // Add the calling convention
7562  Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
7563 
7564  // Add the arguments we omitted previously. The register allocator should
7565  // place these in any free register.
7566  if (IsAnyRegCC)
7567  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
7568  Ops.push_back(getValue(CS.getArgument(i)));
7569 
7570  // Push the arguments from the call instruction up to the register mask.
7571  SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
7572  Ops.append(Call->op_begin() + 2, e);
7573 
7574  // Push live variables for the stack map.
7575  addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
7576 
7577  // Push the register mask info.
7578  if (HasGlue)
7579  Ops.push_back(*(Call->op_end()-2));
7580  else
7581  Ops.push_back(*(Call->op_end()-1));
7582 
7583  // Push the chain (this is originally the first operand of the call, but
7584  // becomes now the last or second to last operand).
7585  Ops.push_back(*(Call->op_begin()));
7586 
7587  // Push the glue flag (last operand).
7588  if (HasGlue)
7589  Ops.push_back(*(Call->op_end()-1));
7590 
7591  SDVTList NodeTys;
7592  if (IsAnyRegCC && HasDef) {
7593  // Create the return types based on the intrinsic definition
7594  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7595  SmallVector<EVT, 3> ValueVTs;
7596  ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7597  assert(ValueVTs.size() == 1 && "Expected only one return value type.");
7598 
7599  // There is always a chain and a glue type at the end
7600  ValueVTs.push_back(MVT::Other);
7601  ValueVTs.push_back(MVT::Glue);
7602  NodeTys = DAG.getVTList(ValueVTs);
7603  } else
7604  NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7605 
7606  // Replace the target specific call node with a PATCHPOINT node.
7607  MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
7608  dl, NodeTys, Ops);
7609 
7610  // Update the NodeMap.
7611  if (HasDef) {
7612  if (IsAnyRegCC)
7613  setValue(CS.getInstruction(), SDValue(MN, 0));
7614  else
7615  setValue(CS.getInstruction(), Result.first);
7616  }
7617 
7618  // Fixup the consumers of the intrinsic. The chain and glue may be used in the
7619  // call sequence. Furthermore the location of the chain and glue can change
7620  // when the AnyReg calling convention is used and the intrinsic returns a
7621  // value.
7622  if (IsAnyRegCC && HasDef) {
7623  SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
7624  SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
7625  DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
7626  } else
7627  DAG.ReplaceAllUsesWith(Call, MN);
7628  DAG.DeleteNode(Call);
7629 
7630  // Inform the Frame Information that we have a patchpoint in this function.
7632 }
7633 
7634 /// Returns an AttributeSet representing the attributes applied to the return
7635 /// value of the given call.
7638  if (CLI.RetSExt)
7639  Attrs.push_back(Attribute::SExt);
7640  if (CLI.RetZExt)
7641  Attrs.push_back(Attribute::ZExt);
7642  if (CLI.IsInReg)
7643  Attrs.push_back(Attribute::InReg);
7644 
7645  return AttributeSet::get(CLI.RetTy->getContext(), AttributeSet::ReturnIndex,
7646  Attrs);
7647 }
7648 
7649 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
7650 /// implementation, which just calls LowerCall.
7651 /// FIXME: When all targets are
7652 /// migrated to using LowerCall, this hook should be integrated into SDISel.
7653 std::pair<SDValue, SDValue>
7655  // Handle the incoming return values from the call.
7656  CLI.Ins.clear();
7657  Type *OrigRetTy = CLI.RetTy;
7658  SmallVector<EVT, 4> RetTys;
7660  auto &DL = CLI.DAG.getDataLayout();
7661  ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
7662 
7664  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
7665 
7666  bool CanLowerReturn =
7667  this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
7668  CLI.IsVarArg, Outs, CLI.RetTy->getContext());
7669 
7670  SDValue DemoteStackSlot;
7671  int DemoteStackIdx = -100;
7672  if (!CanLowerReturn) {
7673  // FIXME: equivalent assert?
7674  // assert(!CS.hasInAllocaArgument() &&
7675  // "sret demotion is incompatible with inalloca");
7676  uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
7677  unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
7679  DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7680  Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
7681 
7682  DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy(DL));
7683  ArgListEntry Entry;
7684  Entry.Node = DemoteStackSlot;
7685  Entry.Ty = StackSlotPtrType;
7686  Entry.isSExt = false;
7687  Entry.isZExt = false;
7688  Entry.isInReg = false;
7689  Entry.isSRet = true;
7690  Entry.isNest = false;
7691  Entry.isByVal = false;
7692  Entry.isReturned = false;
7693  Entry.isSwiftSelf = false;
7694  Entry.isSwiftError = false;
7695  Entry.Alignment = Align;
7696  CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
7697  CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
7698 
7699  // sret demotion isn't compatible with tail-calls, since the sret argument
7700  // points into the callers stack frame.
7701  CLI.IsTailCall = false;
7702  } else {
7703  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7704  EVT VT = RetTys[I];
7705  MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7706  unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7707  for (unsigned i = 0; i != NumRegs; ++i) {
7708  ISD::InputArg MyFlags;
7709  MyFlags.VT = RegisterVT;
7710  MyFlags.ArgVT = VT;
7711  MyFlags.Used = CLI.IsReturnValueUsed;
7712  if (CLI.RetSExt)
7713  MyFlags.Flags.setSExt();
7714  if (CLI.RetZExt)
7715  MyFlags.Flags.setZExt();
7716  if (CLI.IsInReg)
7717  MyFlags.Flags.setInReg();
7718  CLI.Ins.push_back(MyFlags);
7719  }
7720  }
7721  }
7722 
7723  // We push in swifterror return as the last element of CLI.Ins.
7724  ArgListTy &Args = CLI.getArgs();
7725  if (supportSwiftError()) {
7726  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
7727  if (Args[i].isSwiftError) {
7728  ISD::InputArg MyFlags;
7729  MyFlags.VT = getPointerTy(DL);
7730  MyFlags.ArgVT = EVT(getPointerTy(DL));
7731  MyFlags.Flags.setSwiftError();
7732  CLI.Ins.push_back(MyFlags);
7733  }
7734  }
7735  }
7736 
7737  // Handle all of the outgoing arguments.
7738  CLI.Outs.clear();
7739  CLI.OutVals.clear();
7740  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
7741  SmallVector<EVT, 4> ValueVTs;
7742  ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
7743  Type *FinalType = Args[i].Ty;
7744  if (Args[i].isByVal)
7745  FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
7746  bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
7747  FinalType, CLI.CallConv, CLI.IsVarArg);
7748  for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
7749  ++Value) {
7750  EVT VT = ValueVTs[Value];
7751  Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
7752  SDValue Op = SDValue(Args[i].Node.getNode(),
7753  Args[i].Node.getResNo() + Value);
7755  unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
7756 
7757  if (Args[i].isZExt)
7758  Flags.setZExt();
7759  if (Args[i].isSExt)
7760  Flags.setSExt();
7761  if (Args[i].isInReg) {
7762  // If we are using vectorcall calling convention, a structure that is
7763  // passed InReg - is surely an HVA
7765  isa<StructType>(FinalType)) {
7766  // The first value of a structure is marked
7767  if (0 == Value)
7768  Flags.setHvaStart();
7769  Flags.setHva();
7770  }
7771  // Set InReg Flag
7772  Flags.setInReg();
7773  }
7774  if (Args[i].isSRet)
7775  Flags.setSRet();
7776  if (Args[i].isSwiftSelf)
7777  Flags.setSwiftSelf();
7778  if (Args[i].isSwiftError)
7779  Flags.setSwiftError();
7780  if (Args[i].isByVal)
7781  Flags.setByVal();
7782  if (Args[i].isInAlloca) {
7783  Flags.setInAlloca();
7784  // Set the byval flag for CCAssignFn callbacks that don't know about
7785  // inalloca. This way we can know how many bytes we should've allocated
7786  // and how many bytes a callee cleanup function will pop. If we port
7787  // inalloca to more targets, we'll have to add custom inalloca handling
7788  // in the various CC lowering callbacks.
7789  Flags.setByVal();
7790  }
7791  if (Args[i].isByVal || Args[i].isInAlloca) {
7792  PointerType *Ty = cast<PointerType>(Args[i].Ty);
7793  Type *ElementTy = Ty->getElementType();
7794  Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
7795  // For ByVal, alignment should come from FE. BE will guess if this
7796  // info is not there but there are cases it cannot get right.
7797  unsigned FrameAlign;
7798  if (Args[i].Alignment)
7799  FrameAlign = Args[i].Alignment;
7800  else
7801  FrameAlign = getByValTypeAlignment(ElementTy, DL);
7802  Flags.setByValAlign(FrameAlign);
7803  }
7804  if (Args[i].isNest)
7805  Flags.setNest();
7806  if (NeedsRegBlock)
7807  Flags.setInConsecutiveRegs();
7808  Flags.setOrigAlign(OriginalAlignment);
7809 
7810  MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT);
7811  unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT);
7812  SmallVector<SDValue, 4> Parts(NumParts);
7813  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
7814 
7815  if (Args[i].isSExt)
7816  ExtendKind = ISD::SIGN_EXTEND;
7817  else if (Args[i].isZExt)
7818  ExtendKind = ISD::ZERO_EXTEND;
7819 
7820  // Conservatively only handle 'returned' on non-vectors for now
7821  if (Args[i].isReturned && !Op.getValueType().isVector()) {
7822  assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
7823  "unexpected use of 'returned'");
7824  // Before passing 'returned' to the target lowering code, ensure that
7825  // either the register MVT and the actual EVT are the same size or that
7826  // the return value and argument are extended in the same way; in these
7827  // cases it's safe to pass the argument register value unchanged as the
7828  // return register value (although it's at the target's option whether
7829  // to do so)
7830  // TODO: allow code generation to take advantage of partially preserved
7831  // registers rather than clobbering the entire register when the
7832  // parameter extension method is not compatible with the return
7833  // extension method
7834  if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
7835  (ExtendKind != ISD::ANY_EXTEND &&
7836  CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt))
7837  Flags.setReturned();
7838  }
7839 
7840  getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
7841  CLI.CS ? CLI.CS->getInstruction() : nullptr, ExtendKind);
7842 
7843  for (unsigned j = 0; j != NumParts; ++j) {
7844  // if it isn't first piece, alignment must be 1
7845  ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
7846  i < CLI.NumFixedArgs,
7847  i, j*Parts[j].getValueType().getStoreSize());
7848  if (NumParts > 1 && j == 0)
7849  MyFlags.Flags.setSplit();
7850  else if (j != 0) {
7851  MyFlags.Flags.setOrigAlign(1);
7852  if (j == NumParts - 1)
7853  MyFlags.Flags.setSplitEnd();
7854  }
7855 
7856  CLI.Outs.push_back(MyFlags);
7857  CLI.OutVals.push_back(Parts[j]);
7858  }
7859 
7860  if (NeedsRegBlock && Value == NumValues - 1)
7861  CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
7862  }
7863  }
7864 
7865  SmallVector<SDValue, 4> InVals;
7866  CLI.Chain = LowerCall(CLI, InVals);
7867 
7868  // Update CLI.InVals to use outside of this function.
7869  CLI.InVals = InVals;
7870 
7871  // Verify that the target's LowerCall behaved as expected.
7872  assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
7873  "LowerCall didn't return a valid chain!");
7874  assert((!CLI.IsTailCall || InVals.empty()) &&
7875  "LowerCall emitted a return value for a tail call!");
7876  assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
7877  "LowerCall didn't emit the correct number of values!");
7878 
7879  // For a tail call, the return value is merely live-out and there aren't
7880  // any nodes in the DAG representing it. Return a special value to
7881  // indicate that a tail call has been emitted and no more Instructions
7882  // should be processed in the current block.
7883  if (CLI.IsTailCall) {
7884  CLI.DAG.setRoot(CLI.Chain);
7885  return std::make_pair(SDValue(), SDValue());
7886  }
7887 
7888 #ifndef NDEBUG
7889  for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
7890  assert(InVals[i].getNode() && "LowerCall emitted a null value!");
7891  assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
7892  "LowerCall emitted a value with the wrong type!");
7893  }
7894 #endif
7895 
7896  SmallVector<SDValue, 4> ReturnValues;
7897  if (!CanLowerReturn) {
7898  // The instruction result is the result of loading from the
7899  // hidden sret parameter.
7900  SmallVector<EVT, 1> PVTs;
7901  Type *PtrRetTy = PointerType::getUnqual(OrigRetTy);
7902 
7903  ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
7904  assert(PVTs.size() == 1 && "Pointers should fit in one register");
7905  EVT PtrVT = PVTs[0];
7906 
7907  unsigned NumValues = RetTys.size();
7908  ReturnValues.resize(NumValues);
7909  SmallVector<SDValue, 4> Chains(NumValues);
7910 
7911  // An aggregate return value cannot wrap around the address space, so
7912  // offsets to its parts don't wrap either.
7914  Flags.setNoUnsignedWrap(true);
7915 
7916  for (unsigned i = 0; i < NumValues; ++i) {
7917  SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
7918  CLI.DAG.getConstant(Offsets[i], CLI.DL,
7919  PtrVT), &Flags);
7920  SDValue L = CLI.DAG.getLoad(
7921  RetTys[i], CLI.DL, CLI.Chain, Add,
7923  DemoteStackIdx, Offsets[i]),
7924  /* Alignment = */ 1);
7925  ReturnValues[i] = L;
7926  Chains[i] = L.getValue(1);
7927  }
7928 
7929  CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
7930  } else {
7931  // Collect the legal value parts into potentially illegal values
7932  // that correspond to the original function's return values.
7933  Optional<ISD::NodeType> AssertOp;
7934  if (CLI.RetSExt)
7935  AssertOp = ISD::AssertSext;
7936  else if (CLI.RetZExt)
7937  AssertOp = ISD::AssertZext;
7938  unsigned CurReg = 0;
7939  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7940  EVT VT = RetTys[I];
7941  MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7942  unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7943 
7944  ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
7945  NumRegs, RegisterVT, VT, nullptr,
7946  AssertOp));
7947  CurReg += NumRegs;
7948  }
7949 
7950  // For a function returning void, there is no return value. We can't create
7951  // such a node, so we just return a null return value in that case. In
7952  // that case, nothing will actually look at the value.
7953  if (ReturnValues.empty())
7954  return std::make_pair(SDValue(), CLI.Chain);
7955  }
7956 
7957  SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
7958  CLI.DAG.getVTList(RetTys), ReturnValues);
7959  return std::make_pair(Res, CLI.Chain);
7960 }
7961 
7964  SelectionDAG &DAG) const {
7965  if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
7966  Results.push_back(Res);
7967 }
7968 
7970  llvm_unreachable("LowerOperation not implemented for this target!");
7971 }
7972 
7973 void
7975  SDValue Op = getNonRegisterValue(V);
7976  assert((Op.getOpcode() != ISD::CopyFromReg ||
7977  cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
7978  "Copy from a reg to the same reg!");
7979  assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
7980 
7981  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7982  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
7983  V->getType());
7984  SDValue Chain = DAG.getEntryNode();
7985 
7986  ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
7988  ? ISD::ANY_EXTEND
7990  RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
7991  PendingExports.push_back(Chain);
7992 }
7993 
7995 
7996 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
7997 /// entry block, return true. This includes arguments used by switches, since
7998 /// the switch may expand into multiple basic blocks.
7999 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
8000  // With FastISel active, we may be splitting blocks, so force creation
8001  // of virtual registers for all non-dead arguments.
8002  if (FastISel)
8003  return A->use_empty();
8004 
8005  const BasicBlock &Entry = A->getParent()->front();
8006  for (const User *U : A->users())
8007  if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
8008  return false; // Use not in entry block.
8009 
8010  return true;
8011 }
8012 
8013 void SelectionDAGISel::LowerArguments(const Function &F) {
8014  SelectionDAG &DAG = SDB->DAG;
8015  SDLoc dl = SDB->getCurSDLoc();
8016  const DataLayout &DL = DAG.getDataLayout();
8018 
8019  if (!FuncInfo->CanLowerReturn) {
8020  // Put in an sret pointer parameter before all the other parameters.
8021  SmallVector<EVT, 1> ValueVTs;
8022  ComputeValueVTs(*TLI, DAG.getDataLayout(),
8023  PointerType::getUnqual(F.getReturnType()), ValueVTs);
8024 
8025  // NOTE: Assuming that a pointer will never break down to more than one VT
8026  // or one register.
8028  Flags.setSRet();
8029  MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
8030  ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
8032  Ins.push_back(RetArg);
8033  }
8034 
8035  // Set up the incoming argument description vector.
8036  unsigned Idx = 1;
8037  for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
8038  I != E; ++I, ++Idx) {
8039  SmallVector<EVT, 4> ValueVTs;
8040  ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
8041  bool isArgValueUsed = !I->use_empty();
8042  unsigned PartBase = 0;
8043  Type *FinalType = I->getType();
8044  if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal))
8045  FinalType = cast<PointerType>(FinalType)->getElementType();
8046  bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
8047  FinalType, F.getCallingConv(), F.isVarArg());
8048  for (unsigned Value = 0, NumValues = ValueVTs.size();
8049  Value != NumValues; ++Value) {
8050  EVT VT = ValueVTs[Value];
8051  Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
8053  unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
8054 
8055  if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt))
8056  Flags.setZExt();
8057  if (F.getAttributes().hasAttribute(Idx, Attribute::SExt))
8058  Flags.setSExt();
8059  if (F.getAttributes().hasAttribute(Idx, Attribute::InReg)) {
8060  // If we are using vectorcall calling convention, a structure that is
8061  // passed InReg - is surely an HVA
8063  isa<StructType>(I->getType())) {
8064  // The first value of a structure is marked
8065  if (0 == Value)
8066  Flags.setHvaStart();
8067  Flags.setHva();
8068  }
8069  // Set InReg Flag
8070  Flags.setInReg();
8071  }
8072  if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet))
8073  Flags.setSRet();
8074  if (F.getAttributes().hasAttribute(Idx, Attribute::SwiftSelf))
8075  Flags.setSwiftSelf();
8076  if (F.getAttributes().hasAttribute(Idx, Attribute::SwiftError))
8077  Flags.setSwiftError();
8078  if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal))
8079  Flags.setByVal();
8080  if (F.getAttributes().hasAttribute(Idx, Attribute::InAlloca)) {
8081  Flags.setInAlloca();
8082  // Set the byval flag for CCAssignFn callbacks that don't know about
8083  // inalloca. This way we can know how many bytes we should've allocated
8084  // and how many bytes a callee cleanup function will pop. If we port
8085  // inalloca to more targets, we'll have to add custom inalloca handling
8086  // in the various CC lowering callbacks.
8087  Flags.setByVal();
8088  }
8090  // IA Interrupt passes frame (1st parameter) by value in the stack.
8091  if (Idx == 1)
8092  Flags.setByVal();
8093  }
8094  if (Flags.isByVal() || Flags.isInAlloca()) {
8095  PointerType *Ty = cast<PointerType>(I->getType());
8096  Type *ElementTy = Ty->getElementType();
8097  Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8098  // For ByVal, alignment should be passed from FE. BE will guess if
8099  // this info is not there but there are cases it cannot get right.
8100  unsigned FrameAlign;
8101  if (F.getParamAlignment(Idx))
8102  FrameAlign = F.getParamAlignment(Idx);
8103  else
8104  FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
8105  Flags.setByValAlign(FrameAlign);
8106  }
8107  if (F.getAttributes().hasAttribute(Idx, Attribute::Nest))
8108  Flags.setNest();
8109  if (NeedsRegBlock)
8110  Flags.setInConsecutiveRegs();
8111  Flags.setOrigAlign(OriginalAlignment);
8112 
8113  MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8114  unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT);
8115  for (unsigned i = 0; i != NumRegs; ++i) {
8116  ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
8117  Idx-1, PartBase+i*RegisterVT.getStoreSize());
8118  if (NumRegs > 1 && i == 0)
8119  MyFlags.Flags.setSplit();
8120  // if it isn't first piece, alignment must be 1
8121  else if (i > 0) {
8122  MyFlags.Flags.setOrigAlign(1);
8123  if (i == NumRegs - 1)
8124  MyFlags.Flags.setSplitEnd();
8125  }
8126  Ins.push_back(MyFlags);
8127  }
8128  if (NeedsRegBlock && Value == NumValues - 1)
8129  Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
8130  PartBase += VT.getStoreSize();
8131  }
8132  }
8133 
8134  // Call the target to set up the argument values.
8135  SmallVector<SDValue, 8> InVals;
8136  SDValue NewRoot = TLI->LowerFormalArguments(
8137  DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
8138 
8139  // Verify that the target's LowerFormalArguments behaved as expected.
8140  assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
8141  "LowerFormalArguments didn't return a valid chain!");
8142  assert(InVals.size() == Ins.size() &&
8143  "LowerFormalArguments didn't emit the correct number of values!");
8144  DEBUG({
8145  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
8146  assert(InVals[i].getNode() &&
8147  "LowerFormalArguments emitted a null value!");
8148  assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
8149  "LowerFormalArguments emitted a value with the wrong type!");
8150  }
8151  });
8152 
8153  // Update the DAG with the new chain value resulting from argument lowering.
8154  DAG.setRoot(NewRoot);
8155 
8156  // Set up the argument values.
8157  unsigned i = 0;
8158  Idx = 1;
8159  if (!FuncInfo->CanLowerReturn) {
8160  // Create a virtual register for the sret pointer, and put in a copy
8161  // from the sret argument into it.
8162  SmallVector<EVT, 1> ValueVTs;
8163  ComputeValueVTs(*TLI, DAG.getDataLayout(),
8164  PointerType::getUnqual(F.getReturnType()), ValueVTs);
8165  MVT VT = ValueVTs[0].getSimpleVT();
8166  MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8167  Optional<ISD::NodeType> AssertOp = None;
8168  SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1,
8169  RegVT, VT, nullptr, AssertOp);
8170 
8172  MachineRegisterInfo& RegInfo = MF.getRegInfo();
8173  unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
8174  FuncInfo->DemoteRegister = SRetReg;
8175  NewRoot =
8176  SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
8177  DAG.setRoot(NewRoot);
8178 
8179  // i indexes lowered arguments. Bump it past the hidden sret argument.
8180  // Idx indexes LLVM arguments. Don't touch it.
8181  ++i;
8182  }
8183 
8184  for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
8185  ++I, ++Idx) {
8186  SmallVector<SDValue, 4> ArgValues;
8187  SmallVector<EVT, 4> ValueVTs;
8188  ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
8189  unsigned NumValues = ValueVTs.size();
8190 
8191  // If this argument is unused then remember its value. It is used to generate
8192  // debugging information.
8193  bool isSwiftErrorArg =
8194  TLI->supportSwiftError() &&
8195  F.getAttributes().hasAttribute(Idx, Attribute::SwiftError);
8196  if (I->use_empty() && NumValues && !isSwiftErrorArg) {
8197  SDB->setUnusedArgValue(&*I, InVals[i]);
8198 
8199  // Also remember any frame index for use in FastISel.
8200  if (FrameIndexSDNode *FI =
8201  dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
8202  FuncInfo->setArgumentFrameIndex(&*I, FI->getIndex());
8203  }
8204 
8205  for (unsigned Val = 0; Val != NumValues; ++Val) {
8206  EVT VT = ValueVTs[Val];
8207  MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8208  unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT);
8209 
8210  // Even an apparant 'unused' swifterror argument needs to be returned. So
8211  // we do generate a copy for it that can be used on return from the
8212  // function.
8213  if (!I->use_empty() || isSwiftErrorArg) {
8214  Optional<ISD::NodeType> AssertOp;
8215  if (F.getAttributes().hasAttribute(Idx, Attribute::SExt))
8216  AssertOp = ISD::AssertSext;
8217  else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt))
8218  AssertOp = ISD::AssertZext;
8219 
8220  ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i],
8221  NumParts, PartVT, VT,
8222  nullptr, AssertOp));
8223  }
8224 
8225  i += NumParts;
8226  }
8227 
8228  // We don't need to do anything else for unused arguments.
8229  if (ArgValues.empty())
8230  continue;
8231 
8232  // Note down frame index.
8233  if (FrameIndexSDNode *FI =
8234  dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
8235  FuncInfo->setArgumentFrameIndex(&*I, FI->getIndex());
8236 
8237  SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
8238  SDB->getCurSDLoc());
8239 
8240  SDB->setValue(&*I, Res);
8241  if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
8242  if (LoadSDNode *LNode =
8243  dyn_cast<LoadSDNode>(Res.getOperand(0).getNode()))
8244  if (FrameIndexSDNode *FI =
8245  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
8246  FuncInfo->setArgumentFrameIndex(&*I, FI->getIndex());
8247  }
8248 
8249  // Update the SwiftErrorVRegDefMap.
8250  if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
8251  unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8254  FuncInfo->SwiftErrorArg, Reg);
8255  }
8256 
8257  // If this argument is live outside of the entry block, insert a copy from
8258  // wherever we got it to the vreg that other BB's will reference it as.
8259  if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
8260  // If we can, though, try to skip creating an unnecessary vreg.
8261  // FIXME: This isn't very clean... it would be nice to make this more
8262  // general. It's also subtly incompatible with the hacks FastISel
8263  // uses with vregs.
8264  unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8266  FuncInfo->ValueMap[&*I] = Reg;
8267  continue;
8268  }
8269  }
8273  }
8274  }
8275 
8276  assert(i == InVals.size() && "Argument register count mismatch!");
8277 
8278  // Finally, if the target has anything special to do, allow it to do so.
8280 }
8281 
8282 /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
8283 /// ensure constants are generated when needed. Remember the virtual registers
8284 /// that need to be added to the Machine PHI nodes as input. We cannot just
8285 /// directly add them, because expansion might result in multiple MBB's for one
8286 /// BB. As such, the start of the BB might correspond to a different MBB than
8287 /// the end.
8288 ///
8289 void
8290 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
8291  const TerminatorInst *TI = LLVMBB->getTerminator();
8292 
8294 
8295  // Check PHI nodes in successors that expect a value to be available from this
8296  // block.
8297  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
8298  const BasicBlock *SuccBB = TI->getSuccessor(succ);
8299  if (!isa<PHINode>(SuccBB->begin())) continue;
8300  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
8301 
8302  // If this terminator has multiple identical successors (common for
8303  // switches), only handle each succ once.
8304  if (!SuccsHandled.insert(SuccMBB).second)
8305  continue;
8306 
8307  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
8308 
8309  // At this point we know that there is a 1-1 correspondence between LLVM PHI
8310  // nodes and Machine PHI nodes, but the incoming operands have not been
8311  // emitted yet.
8312  for (BasicBlock::const_iterator I = SuccBB->begin();
8313  const PHINode *PN = dyn_cast<PHINode>(I); ++I) {
8314  // Ignore dead phi's.
8315  if (PN->use_empty()) continue;
8316 
8317  // Skip empty types
8318  if (PN->getType()->isEmptyTy())
8319  continue;
8320 
8321  unsigned Reg;
8322  const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
8323 
8324  if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
8325  unsigned &RegOut = ConstantsOut[C];
8326  if (RegOut == 0) {
8327  RegOut = FuncInfo.CreateRegs(C->getType());
8328  CopyValueToVirtualRegister(C, RegOut);
8329  }
8330  Reg = RegOut;
8331  } else {
8333  FuncInfo.ValueMap.find(PHIOp);
8334  if (I != FuncInfo.ValueMap.end())
8335  Reg = I->second;
8336  else {
8337  assert(isa<AllocaInst>(PHIOp) &&
8338  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
8339  "Didn't codegen value into a register!??");
8340  Reg = FuncInfo.CreateRegs(PHIOp->getType());
8341  CopyValueToVirtualRegister(PHIOp, Reg);
8342  }
8343  }
8344 
8345  // Remember that this register needs to added to the machine PHI node as
8346  // the input for this MBB.
8347  SmallVector<EVT, 4> ValueVTs;
8348  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8349  ComputeValueVTs(TLI, DAG.getDataLayout(), PN->getType(), ValueVTs);
8350  for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
8351  EVT VT = ValueVTs[vti];
8352  unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
8353  for (unsigned i = 0, e = NumRegisters; i != e; ++i)
8354  FuncInfo.PHINodesToUpdate.push_back(
8355  std::make_pair(&*MBBI++, Reg + i));
8356  Reg += NumRegisters;
8357  }
8358  }
8359  }
8360 
8361  ConstantsOut.clear();
8362 }
8363 
8364 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
8365 /// is 0.
8367 SelectionDAGBuilder::StackProtectorDescriptor::
8368 AddSuccessorMBB(const BasicBlock *BB,
8369  MachineBasicBlock *ParentMBB,
8370  bool IsLikely,
8371  MachineBasicBlock *SuccMBB) {
8372  // If SuccBB has not been created yet, create it.
8373  if (!SuccMBB) {
8374  MachineFunction *MF = ParentMBB->getParent();
8375  MachineFunction::iterator BBI(ParentMBB);
8376  SuccMBB = MF->CreateMachineBasicBlock(BB);
8377  MF->insert(++BBI, SuccMBB);
8378  }
8379  // Add it as a successor of ParentMBB.
8380  ParentMBB->addSuccessor(
8382  return SuccMBB;
8383 }
8384 
8385 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
8387  if (++I == FuncInfo.MF->end())
8388  return nullptr;
8389  return &*I;
8390 }
8391 
8392 /// During lowering new call nodes can be created (such as memset, etc.).
8393 /// Those will become new roots of the current DAG, but complications arise
8394 /// when they are tail calls. In such cases, the call lowering will update
8395 /// the root, but the builder still needs to know that a tail call has been
8396 /// lowered in order to avoid generating an additional return.
8397 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
8398  // If the node is null, we do have a tail call.
8399  if (MaybeTC.getNode() != nullptr)
8400  DAG.setRoot(MaybeTC);
8401  else
8402  HasTailCall = true;
8403 }
8404 
8405 bool SelectionDAGBuilder::isDense(const CaseClusterVector &Clusters,
8406  const SmallVectorImpl<unsigned> &TotalCases,
8407  unsigned First, unsigned Last,
8408  unsigned Density) const {
8409  assert(Last >= First);
8410  assert(TotalCases[Last] >= TotalCases[First]);
8411 
8412  const APInt &LowCase = Clusters[First].Low->getValue();
8413  const APInt &HighCase = Clusters[Last].High->getValue();
8414  assert(LowCase.getBitWidth() == HighCase.getBitWidth());
8415 
8416  // FIXME: A range of consecutive cases has 100% density, but only requires one
8417  // comparison to lower. We should discriminate against such consecutive ranges
8418  // in jump tables.
8419 
8420  uint64_t Diff = (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100);
8421  uint64_t Range = Diff + 1;
8422 
8423  uint64_t NumCases =
8424  TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
8425 
8426  assert(NumCases < UINT64_MAX / 100);
8427  assert(Range >= NumCases);
8428 
8429  return NumCases * 100 >= Range * Density;
8430 }
8431 
8432 static inline bool areJTsAllowed(const TargetLowering &TLI,
8433  const SwitchInst *SI) {
8434  const Function *Fn = SI->getParent()->getParent();
8435  if (Fn->getFnAttribute("no-jump-tables").getValueAsString() == "true")
8436  return false;
8437 
8440 }
8441 
8442 bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
8443  unsigned First, unsigned Last,
8444  const SwitchInst *SI,
8445  MachineBasicBlock *DefaultMBB,
8446  CaseCluster &JTCluster) {
8447  assert(First <= Last);
8448 
8449  auto Prob = BranchProbability::getZero();
8450  unsigned NumCmps = 0;
8451  std::vector<MachineBasicBlock*> Table;
8453 
8454  // Initialize probabilities in JTProbs.
8455  for (unsigned I = First; I <= Last; ++I)
8456  JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
8457 
8458  for (unsigned I = First; I <= Last; ++I) {
8459  assert(Clusters[I].Kind == CC_Range);
8460  Prob += Clusters[I].Prob;
8461  const APInt &Low = Clusters[I].Low->getValue();
8462  const APInt &High = Clusters[I].High->getValue();
8463  NumCmps += (Low == High) ? 1 : 2;
8464  if (I != First) {
8465  // Fill the gap between this and the previous cluster.
8466  const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
8467  assert(PreviousHigh.slt(Low));
8468  uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
8469  for (uint64_t J = 0; J < Gap; J++)
8470  Table.push_back(DefaultMBB);
8471  }
8472  uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
8473  for (uint64_t J = 0; J < ClusterSize; ++J)
8474  Table.push_back(Clusters[I].MBB);
8475  JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
8476  }
8477 
8478  unsigned NumDests = JTProbs.size();
8479  if (isSuitableForBitTests(NumDests, NumCmps,
8480  Clusters[First].Low->getValue(),
8481  Clusters[Last].High->getValue())) {
8482  // Clusters[First..Last] should be lowered as bit tests instead.
8483  return false;
8484  }
8485 
8486  // Create the MBB that will load from and jump through the table.
8487  // Note: We create it here, but it's not inserted into the function yet.
8488  MachineFunction *CurMF = FuncInfo.MF;
8489  MachineBasicBlock *JumpTableMBB =
8490  CurMF->CreateMachineBasicBlock(SI->getParent());
8491 
8492  // Add successors. Note: use table order for determinism.
8494  for (MachineBasicBlock *Succ : Table) {
8495  if (Done.count(Succ))
8496  continue;
8497  addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
8498  Done.insert(Succ);
8499  }
8500  JumpTableMBB->normalizeSuccProbs();
8501 
8502  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8503  unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
8504  ->createJumpTableIndex(Table);
8505 
8506  // Set up the jump table info.
8507  JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
8508  JumpTableHeader JTH(Clusters[First].Low->getValue(),
8509  Clusters[Last].High->getValue(), SI->getCondition(),
8510  nullptr, false);
8511  JTCases.emplace_back(std::move(JTH), std::move(JT));
8512 
8513  JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
8514  JTCases.size() - 1, Prob);
8515  return true;
8516 }
8517 
8518 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
8519  const SwitchInst *SI,
8520  MachineBasicBlock *DefaultMBB) {
8521 #ifndef NDEBUG
8522  // Clusters must be non-empty, sorted, and only contain Range clusters.
8523  assert(!Clusters.empty());
8524  for (CaseCluster &C : Clusters)
8525  assert(C.Kind == CC_Range);
8526  for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
8527  assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
8528 #endif
8529 
8530  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8531  if (!areJTsAllowed(TLI, SI))
8532  return;
8533 
8534  const bool OptForSize = DefaultMBB->getParent()->getFunction()->optForSize();
8535 
8536  const int64_t N = Clusters.size();
8537  const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
8538  const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
8539  const unsigned MaxJumpTableSize =
8540  OptForSize || TLI.getMaximumJumpTableSize() == 0
8541  ? UINT_MAX : TLI.getMaximumJumpTableSize();
8542 
8543  if (N < 2 || N < MinJumpTableEntries)
8544  return;
8545 
8546  // TotalCases[i]: Total nbr of cases in Clusters[0..i].
8547  SmallVector<unsigned, 8> TotalCases(N);
8548  for (unsigned i = 0; i < N; ++i) {
8549  const APInt &Hi = Clusters[i].High->getValue();
8550  const APInt &Lo = Clusters[i].Low->getValue();
8551  TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
8552  if (i != 0)
8553  TotalCases[i] += TotalCases[i - 1];
8554  }
8555 
8556  const unsigned MinDensity =
8558 
8559  // Cheap case: the whole range may be suitable for jump table.
8560  unsigned JumpTableSize = (Clusters[N - 1].High->getValue() -
8561  Clusters[0].Low->getValue())
8562  .getLimitedValue(UINT_MAX - 1) + 1;
8563  if (JumpTableSize <= MaxJumpTableSize &&
8564  isDense(Clusters, TotalCases, 0, N - 1, MinDensity)) {
8565  CaseCluster JTCluster;
8566  if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
8567  Clusters[0] = JTCluster;
8568  Clusters.resize(1);
8569  return;
8570  }
8571  }
8572 
8573  // The algorithm below is not suitable for -O0.
8574  if (TM.getOptLevel() == CodeGenOpt::None)
8575  return;
8576 
8577  // Split Clusters into minimum number of dense partitions. The algorithm uses
8578  // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
8579  // for the Case Statement'" (1994), but builds the MinPartitions array in
8580  // reverse order to make it easier to reconstruct the partitions in ascending
8581  // order. In the choice between two optimal partitionings, it picks the one
8582  // which yields more jump tables.
8583 
8584  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
8585  SmallVector<unsigned, 8> MinPartitions(N);
8586  // LastElement[i] is the last element of the partition starting at i.
8587  SmallVector<unsigned, 8> LastElement(N);
8588  // PartitionsScore[i] is used to break ties when choosing between two
8589  // partitionings resulting in the same number of partitions.
8590  SmallVector<unsigned, 8> PartitionsScore(N);
8591  // For PartitionsScore, a small number of comparisons is considered as good as
8592  // a jump table and a single comparison is considered better than a jump
8593  // table.
8594  enum PartitionScores : unsigned {
8595  NoTable = 0,
8596  Table = 1,
8597  FewCases = 1,
8598  SingleCase = 2
8599  };
8600 
8601  // Base case: There is only one way to partition Clusters[N-1].
8602  MinPartitions[N - 1] = 1;
8603  LastElement[N - 1] = N - 1;
8604  PartitionsScore[N - 1] = PartitionScores::SingleCase;
8605 
8606  // Note: loop indexes are signed to avoid underflow.
8607  for (int64_t i = N - 2; i >= 0; i--) {
8608  // Find optimal partitioning of Clusters[i..N-1].
8609  // Baseline: Put Clusters[i] into a partition on its own.
8610  MinPartitions[i] = MinPartitions[i + 1] + 1;
8611  LastElement[i] = i;
8612  PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
8613 
8614  // Search for a solution that results in fewer partitions.
8615  for (int64_t j = N - 1; j > i; j--) {
8616  // Try building a partition from Clusters[i..j].
8617  JumpTableSize = (Clusters[j].High->getValue() -
8618  Clusters[i].Low->getValue())
8619  .getLimitedValue(UINT_MAX - 1) + 1;
8620  if (JumpTableSize <= MaxJumpTableSize &&
8621  isDense(Clusters, TotalCases, i, j, MinDensity)) {
8622  unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
8623  unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
8624  int64_t NumEntries = j - i + 1;
8625 
8626  if (NumEntries == 1)
8627  Score += PartitionScores::SingleCase;
8628  else if (NumEntries <= SmallNumberOfEntries)
8629  Score += PartitionScores::FewCases;
8630  else if (NumEntries >= MinJumpTableEntries)
8631  Score += PartitionScores::Table;
8632 
8633  // If this leads to fewer partitions, or to the same number of
8634  // partitions with better score, it is a better partitioning.
8635  if (NumPartitions < MinPartitions[i] ||
8636  (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
8637  MinPartitions[i] = NumPartitions;
8638  LastElement[i] = j;
8639  PartitionsScore[i] = Score;
8640  }
8641  }
8642  }
8643  }
8644 
8645  // Iterate over the partitions, replacing some with jump tables in-place.
8646  unsigned DstIndex = 0;
8647  for (unsigned First = 0, Last; First < N; First = Last + 1) {
8648  Last = LastElement[First];
8649  assert(Last >= First);
8650  assert(DstIndex <= First);
8651  unsigned NumClusters = Last - First + 1;
8652 
8653  CaseCluster JTCluster;
8654  if (NumClusters >= MinJumpTableEntries &&
8655  buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
8656  Clusters[DstIndex++] = JTCluster;
8657  } else {
8658  for (unsigned I = First; I <= Last; ++I)
8659  std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
8660  }
8661  }
8662  Clusters.resize(DstIndex);
8663 }
8664 
8665 bool SelectionDAGBuilder::rangeFitsInWord(const APInt &Low, const APInt &High) {
8666  // FIXME: Using the pointer type doesn't seem ideal.
8667  uint64_t BW = DAG.getDataLayout().getPointerSizeInBits();
8668  uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX - 1) + 1;
8669  return Range <= BW;
8670 }
8671 
8672 bool SelectionDAGBuilder::isSuitableForBitTests(unsigned NumDests,
8673  unsigned NumCmps,
8674  const APInt &Low,
8675  const APInt &High) {
8676  // FIXME: I don't think NumCmps is the correct metric: a single case and a
8677  // range of cases both require only one branch to lower. Just looking at the
8678  // number of clusters and destinations should be enough to decide whether to
8679  // build bit tests.
8680 
8681  // To lower a range with bit tests, the range must fit the bitwidth of a
8682  // machine word.
8683  if (!rangeFitsInWord(Low, High))
8684  return false;
8685 
8686  // Decide whether it's profitable to lower this range with bit tests. Each
8687  // destination requires a bit test and branch, and there is an overall range
8688  // check branch. For a small number of clusters, separate comparisons might be
8689  // cheaper, and for many destinations, splitting the range might be better.
8690  return (NumDests == 1 && NumCmps >= 3) ||
8691  (NumDests == 2 && NumCmps >= 5) ||
8692  (NumDests == 3 && NumCmps >= 6);
8693 }
8694 
8695 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
8696  unsigned First, unsigned Last,
8697  const SwitchInst *SI,
8698  CaseCluster &BTCluster) {
8699  assert(First <= Last);
8700  if (First == Last)
8701  return false;
8702 
8703  BitVector Dests(FuncInfo.MF->getNumBlockIDs());
8704  unsigned NumCmps = 0;
8705  for (int64_t I = First; I <= Last; ++I) {
8706  assert(Clusters[I].Kind == CC_Range);
8707  Dests.set(Clusters[I].MBB->getNumber());
8708  NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
8709  }
8710  unsigned NumDests = Dests.count();
8711 
8712  APInt Low = Clusters[First].Low->getValue();
8713  APInt High = Clusters[Last].High->getValue();
8714  assert(Low.slt(High));
8715 
8716  if (!isSuitableForBitTests(NumDests, NumCmps, Low, High))
8717  return false;
8718 
8719  APInt LowBound;
8720  APInt CmpRange;
8721 
8722  const int BitWidth = DAG.getTargetLoweringInfo()
8723  .getPointerTy(DAG.getDataLayout())
8724  .getSizeInBits();
8725  assert(rangeFitsInWord(Low, High) && "Case range must fit in bit mask!");
8726 
8727  // Check if the clusters cover a contiguous range such that no value in the
8728  // range will jump to the default statement.
8729  bool ContiguousRange = true;
8730  for (int64_t I = First + 1; I <= Last; ++I) {
8731  if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
8732  ContiguousRange = false;
8733  break;
8734  }
8735  }
8736 
8737  if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
8738  // Optimize the case where all the case values fit in a word without having
8739  // to subtract minValue. In this case, we can optimize away the subtraction.
8740  LowBound = APInt::getNullValue(Low.getBitWidth());
8741  CmpRange = High;
8742  ContiguousRange = false;
8743  } else {
8744  LowBound = Low;
8745  CmpRange = High - Low;
8746  }
8747 
8748  CaseBitsVector CBV;
8749  auto TotalProb = BranchProbability::getZero();
8750  for (unsigned i = First; i <= Last; ++i) {
8751  // Find the CaseBits for this destination.
8752  unsigned j;
8753  for (j = 0; j < CBV.size(); ++j)
8754  if (CBV[j].BB == Clusters[i].MBB)
8755  break;
8756  if (j == CBV.size())
8757  CBV.push_back(
8758  CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
8759  CaseBits *CB = &CBV[j];
8760 
8761  // Update Mask, Bits and ExtraProb.
8762  uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
8763  uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
8764  assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
8765  CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
8766  CB->Bits += Hi - Lo + 1;
8767  CB->ExtraProb += Clusters[i].Prob;
8768  TotalProb += Clusters[i].Prob;
8769  }
8770 
8771  BitTestInfo BTI;
8772  std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
8773  // Sort by probability first, number of bits second.
8774  if (a.ExtraProb != b.ExtraProb)
8775  return a.ExtraProb > b.ExtraProb;
8776  return a.Bits > b.Bits;
8777  });
8778 
8779  for (auto &CB : CBV) {
8780  MachineBasicBlock *BitTestBB =
8782  BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
8783  }
8784  BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
8785  SI->getCondition(), -1U, MVT::Other, false,
8786  ContiguousRange, nullptr, nullptr, std::move(BTI),
8787  TotalProb);
8788 
8789  BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
8790  BitTestCases.size() - 1, TotalProb);
8791  return true;
8792 }
8793 
8794 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
8795  const SwitchInst *SI) {
8796 // Partition Clusters into as few subsets as possible, where each subset has a
8797 // range that fits in a machine word and has <= 3 unique destinations.
8798 
8799 #ifndef NDEBUG
8800  // Clusters must be sorted and contain Range or JumpTable clusters.
8801  assert(!Clusters.empty());
8802  assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
8803  for (const CaseCluster &C : Clusters)
8804  assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
8805  for (unsigned i = 1; i < Clusters.size(); ++i)
8806  assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
8807 #endif
8808 
8809  // The algorithm below is not suitable for -O0.
8810  if (TM.getOptLevel() == CodeGenOpt::None)
8811  return;
8812 
8813  // If target does not have legal shift left, do not emit bit tests at all.
8814  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8815  EVT PTy = TLI.getPointerTy(DAG.getDataLayout());
8816  if (!TLI.isOperationLegal(ISD::SHL, PTy))
8817  return;
8818 
8819  int BitWidth = PTy.getSizeInBits();
8820  const int64_t N = Clusters.size();
8821 
8822  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
8823  SmallVector<unsigned, 8> MinPartitions(N);
8824  // LastElement[i] is the last element of the partition starting at i.
8825  SmallVector<unsigned, 8> LastElement(N);
8826 
8827  // FIXME: This might not be the best algorithm for finding bit test clusters.
8828 
8829  // Base case: There is only one way to partition Clusters[N-1].
8830  MinPartitions[N - 1] = 1;
8831  LastElement[N - 1] = N - 1;
8832 
8833  // Note: loop indexes are signed to avoid underflow.
8834  for (int64_t i = N - 2; i >= 0; --i) {
8835  // Find optimal partitioning of Clusters[i..N-1].
8836  // Baseline: Put Clusters[i] into a partition on its own.
8837  MinPartitions[i] = MinPartitions[i + 1] + 1;
8838  LastElement[i] = i;
8839 
8840  // Search for a solution that results in fewer partitions.
8841  // Note: the search is limited by BitWidth, reducing time complexity.
8842  for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
8843  // Try building a partition from Clusters[i..j].
8844 
8845  // Check the range.
8846  if (!rangeFitsInWord(Clusters[i].Low->getValue(),
8847  Clusters[j].High->getValue()))
8848  continue;
8849 
8850  // Check nbr of destinations and cluster types.
8851  // FIXME: This works, but doesn't seem very efficient.
8852  bool RangesOnly = true;
8853  BitVector Dests(FuncInfo.MF->getNumBlockIDs());
8854  for (int64_t k = i; k <= j; k++) {
8855  if (Clusters[k].Kind != CC_Range) {
8856  RangesOnly = false;
8857  break;
8858  }
8859  Dests.set(Clusters[k].MBB->getNumber());
8860  }
8861  if (!RangesOnly || Dests.count() > 3)
8862  break;
8863 
8864  // Check if it's a better partition.
8865  unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
8866  if (NumPartitions < MinPartitions[i]) {
8867  // Found a better partition.
8868  MinPartitions[i] = NumPartitions;
8869  LastElement[i] = j;
8870  }
8871  }
8872  }
8873 
8874  // Iterate over the partitions, replacing with bit-test clusters in-place.
8875  unsigned DstIndex = 0;
8876  for (unsigned First = 0, Last; First < N; First = Last + 1) {
8877  Last = LastElement[First];
8878  assert(First <= Last);
8879  assert(DstIndex <= First);
8880 
8881  CaseCluster BitTestCluster;
8882  if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
8883  Clusters[DstIndex++] = BitTestCluster;
8884  } else {
8885  size_t NumClusters = Last - First + 1;
8886  std::memmove(&Clusters[DstIndex], &Clusters[First],
8887  sizeof(Clusters[0]) * NumClusters);
8888  DstIndex += NumClusters;
8889  }
8890  }
8891  Clusters.resize(DstIndex);
8892 }
8893 
8894 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
8895  MachineBasicBlock *SwitchMBB,
8896  MachineBasicBlock *DefaultMBB) {
8897  MachineFunction *CurMF = FuncInfo.MF;
8898  MachineBasicBlock *NextMBB = nullptr;
8899  MachineFunction::iterator BBI(W.MBB);
8900  if (++BBI != FuncInfo.MF->end())
8901  NextMBB = &*BBI;
8902 
8903  unsigned Size = W.LastCluster - W.FirstCluster + 1;
8904 
8906 
8907  if (Size == 2 && W.MBB == SwitchMBB) {
8908  // If any two of the cases has the same destination, and if one value
8909  // is the same as the other, but has one bit unset that the other has set,
8910  // use bit manipulation to do two compares at once. For example:
8911  // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
8912  // TODO: This could be extended to merge any 2 cases in switches with 3
8913  // cases.
8914  // TODO: Handle cases where W.CaseBB != SwitchBB.
8915  CaseCluster &Small = *W.FirstCluster;
8916  CaseCluster &Big = *W.LastCluster;
8917 
8918  if (Small.Low == Small.High && Big.Low == Big.High &&
8919  Small.MBB == Big.MBB) {
8920  const APInt &SmallValue = Small.Low->getValue();
8921  const APInt &BigValue = Big.Low->getValue();
8922 
8923  // Check that there is only one bit different.
8924  APInt CommonBit = BigValue ^ SmallValue;
8925  if (CommonBit.isPowerOf2()) {
8926  SDValue CondLHS = getValue(Cond);
8927  EVT VT = CondLHS.getValueType();
8928  SDLoc DL = getCurSDLoc();
8929 
8930  SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
8931  DAG.getConstant(CommonBit, DL, VT));
8932  SDValue Cond = DAG.getSetCC(
8933  DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
8934  ISD::SETEQ);
8935 
8936  // Update successor info.
8937  // Both Small and Big will jump to Small.BB, so we sum up the
8938  // probabilities.
8939  addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
8940  if (BPI)
8941  addSuccessorWithProb(
8942  SwitchMBB, DefaultMBB,
8943  // The default destination is the first successor in IR.
8944  BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
8945  else
8946  addSuccessorWithProb(SwitchMBB, DefaultMBB);
8947 
8948  // Insert the true branch.
8949  SDValue BrCond =
8950  DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
8951  DAG.getBasicBlock(Small.MBB));
8952  // Insert the false branch.
8953  BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
8954  DAG.getBasicBlock(DefaultMBB));
8955 
8956  DAG.setRoot(BrCond);
8957  return;
8958  }
8959  }
8960  }
8961 
8962  if (TM.getOptLevel() != CodeGenOpt::None) {
8963  // Order cases by probability so the most likely case will be checked first.
8964  std::sort(W.FirstCluster, W.LastCluster + 1,
8965  [](const CaseCluster &a, const CaseCluster &b) {
8966  return a.Prob > b.Prob;
8967  });
8968 
8969  // Rearrange the case blocks so that the last one falls through if possible
8970  // without without changing the order of probabilities.
8971  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
8972  --I;
8973  if (I->Prob > W.LastCluster->Prob)
8974  break;
8975  if (I->Kind == CC_Range && I->MBB == NextMBB) {
8976  std::swap(*I, *W.LastCluster);
8977  break;
8978  }
8979  }
8980  }
8981 
8982  // Compute total probability.
8983  BranchProbability DefaultProb = W.DefaultProb;
8984  BranchProbability UnhandledProbs = DefaultProb;
8985  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
8986  UnhandledProbs += I->Prob;
8987 
8988  MachineBasicBlock *CurMBB = W.MBB;
8989  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
8990  MachineBasicBlock *Fallthrough;
8991  if (I == W.LastCluster) {
8992  // For the last cluster, fall through to the default destination.
8993  Fallthrough = DefaultMBB;
8994  } else {
8995  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
8996  CurMF->insert(BBI, Fallthrough);
8997  // Put Cond in a virtual register to make it available from the new blocks.
8998  ExportFromCurrentBlock(Cond);
8999  }
9000  UnhandledProbs -= I->Prob;
9001 
9002  switch (I->Kind) {
9003  case CC_JumpTable: {
9004  // FIXME: Optimize away range check based on pivot comparisons.
9005  JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
9006  JumpTable *JT = &JTCases[I->JTCasesIndex].second;
9007 
9008  // The jump block hasn't been inserted yet; insert it here.
9009  MachineBasicBlock *JumpMBB = JT->MBB;
9010  CurMF->insert(BBI, JumpMBB);
9011 
9012  auto JumpProb = I->Prob;
9013  auto FallthroughProb = UnhandledProbs;
9014 
9015  // If the default statement is a target of the jump table, we evenly
9016  // distribute the default probability to successors of CurMBB. Also
9017  // update the probability on the edge from JumpMBB to Fallthrough.
9018  for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
9019  SE = JumpMBB->succ_end();
9020  SI != SE; ++SI) {
9021  if (*SI == DefaultMBB) {
9022  JumpProb += DefaultProb / 2;
9023  FallthroughProb -= DefaultProb / 2;
9024  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
9025  JumpMBB->normalizeSuccProbs();
9026  break;
9027  }
9028  }
9029 
9030  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
9031  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
9032  CurMBB->normalizeSuccProbs();
9033 
9034  // The jump table header will be inserted in our current block, do the
9035  // range check, and fall through to our fallthrough block.
9036  JTH->HeaderBB = CurMBB;
9037  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
9038 
9039  // If we're in the right place, emit the jump table header right now.
9040  if (CurMBB == SwitchMBB) {
9041  visitJumpTableHeader(*JT, *JTH, SwitchMBB);
9042  JTH->Emitted = true;
9043  }
9044  break;
9045  }
9046  case CC_BitTests: {
9047  // FIXME: Optimize away range check based on pivot comparisons.
9048  BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
9049 
9050  // The bit test blocks haven't been inserted yet; insert them here.
9051  for (BitTestCase &BTC : BTB->Cases)
9052  CurMF->insert(BBI, BTC.ThisBB);
9053 
9054  // Fill in fields of the BitTestBlock.
9055  BTB->Parent = CurMBB;
9056  BTB->Default = Fallthrough;
9057 
9058  BTB->DefaultProb = UnhandledProbs;
9059  // If the cases in bit test don't form a contiguous range, we evenly
9060  // distribute the probability on the edge to Fallthrough to two
9061  // successors of CurMBB.
9062  if (!BTB->ContiguousRange) {
9063  BTB->Prob += DefaultProb / 2;
9064  BTB->DefaultProb -= DefaultProb / 2;
9065  }
9066 
9067  // If we're in the right place, emit the bit test header right now.
9068  if (CurMBB == SwitchMBB) {
9069  visitBitTestHeader(*BTB, SwitchMBB);
9070  BTB->Emitted = true;
9071  }
9072  break;
9073  }
9074  case CC_Range: {
9075  const Value *RHS, *LHS, *MHS;
9076  ISD::CondCode CC;
9077  if (I->Low == I->High) {
9078  // Check Cond == I->Low.
9079  CC = ISD::SETEQ;
9080  LHS = Cond;
9081  RHS=I->Low;
9082  MHS = nullptr;
9083  } else {
9084  // Check I->Low <= Cond <= I->High.
9085  CC = ISD::SETLE;
9086  LHS = I->Low;
9087  MHS = Cond;
9088  RHS = I->High;
9089  }
9090 
9091  // The false probability is the sum of all unhandled cases.
9092  CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Prob,
9093  UnhandledProbs);
9094 
9095  if (CurMBB == SwitchMBB)
9096  visitSwitchCase(CB, SwitchMBB);
9097  else
9098  SwitchCases.push_back(CB);
9099 
9100  break;
9101  }
9102  }
9103  CurMBB = Fallthrough;
9104  }
9105 }
9106 
9107 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
9108  CaseClusterIt First,
9109  CaseClusterIt Last) {
9110  return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
9111  if (X.Prob != CC.Prob)
9112  return X.Prob > CC.Prob;
9113 
9114  // Ties are broken by comparing the case value.
9115  return X.Low->getValue().slt(CC.Low->getValue());
9116  });
9117 }
9118 
9119 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
9120  const SwitchWorkListItem &W,
9121  Value *Cond,
9122  MachineBasicBlock *SwitchMBB) {
9123  assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
9124  "Clusters not sorted?");
9125 
9126  assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
9127 
9128  // Balance the tree based on branch probabilities to create a near-optimal (in
9129  // terms of search time given key frequency) binary search tree. See e.g. Kurt
9130  // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
9131  CaseClusterIt LastLeft = W.FirstCluster;
9132  CaseClusterIt FirstRight = W.LastCluster;
9133  auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
9134  auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
9135 
9136  // Move LastLeft and FirstRight towards each other from opposite directions to
9137  // find a partitioning of the clusters which balances the probability on both
9138  // sides. If LeftProb and RightProb are equal, alternate which side is
9139  // taken to ensure 0-probability nodes are distributed evenly.
9140  unsigned I = 0;
9141  while (LastLeft + 1 < FirstRight) {
9142  if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
9143  LeftProb += (++LastLeft)->Prob;
9144  else
9145  RightProb += (--FirstRight)->Prob;
9146  I++;
9147  }
9148 
9149  for (;;) {
9150  // Our binary search tree differs from a typical BST in that ours can have up
9151  // to three values in each leaf. The pivot selection above doesn't take that
9152  // into account, which means the tree might require more nodes and be less
9153  // efficient. We compensate for this here.
9154 
9155  unsigned NumLeft = LastLeft - W.FirstCluster + 1;
9156  unsigned NumRight = W.LastCluster - FirstRight + 1;
9157 
9158  if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
9159  // If one side has less than 3 clusters, and the other has more than 3,
9160  // consider taking a cluster from the other side.
9161 
9162  if (NumLeft < NumRight) {
9163  // Consider moving the first cluster on the right to the left side.
9164  CaseCluster &CC = *FirstRight;
9165  unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9166  unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9167  if (LeftSideRank <= RightSideRank) {
9168  // Moving the cluster to the left does not demote it.
9169  ++LastLeft;
9170  ++FirstRight;
9171  continue;
9172  }
9173  } else {
9174  assert(NumRight < NumLeft);
9175  // Consider moving the last element on the left to the right side.
9176  CaseCluster &CC = *LastLeft;
9177  unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9178  unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9179  if (RightSideRank <= LeftSideRank) {
9180  // Moving the cluster to the right does not demot it.
9181  --LastLeft;
9182  --FirstRight;
9183  continue;
9184  }
9185  }
9186  }
9187  break;
9188  }
9189 
9190  assert(LastLeft + 1 == FirstRight);
9191  assert(LastLeft >= W.FirstCluster);
9192  assert(FirstRight <= W.LastCluster);
9193 
9194  // Use the first element on the right as pivot since we will make less-than
9195  // comparisons against it.
9196  CaseClusterIt PivotCluster = FirstRight;
9197  assert(PivotCluster > W.FirstCluster);
9198  assert(PivotCluster <= W.LastCluster);
9199 
9200  CaseClusterIt FirstLeft = W.FirstCluster;
9201  CaseClusterIt LastRight = W.LastCluster;
9202 
9203  const ConstantInt *Pivot = PivotCluster->Low;
9204 
9205  // New blocks will be inserted immediately after the current one.
9206  MachineFunction::iterator BBI(W.MBB);
9207  ++BBI;
9208 
9209  // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
9210  // we can branch to its destination directly if it's squeezed exactly in
9211  // between the known lower bound and Pivot - 1.
9212  MachineBasicBlock *LeftMBB;
9213  if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
9214  FirstLeft->Low == W.GE &&
9215  (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
9216  LeftMBB = FirstLeft->MBB;
9217  } else {
9218  LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9219  FuncInfo.MF->insert(BBI, LeftMBB);
9220  WorkList.push_back(
9221  {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
9222  // Put Cond in a virtual register to make it available from the new blocks.
9223  ExportFromCurrentBlock(Cond);
9224  }
9225 
9226  // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
9227  // single cluster, RHS.Low == Pivot, and we can branch to its destination
9228  // directly if RHS.High equals the current upper bound.
9229  MachineBasicBlock *RightMBB;
9230  if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
9231  W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
9232  RightMBB = FirstRight->MBB;
9233  } else {
9234  RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9235  FuncInfo.MF->insert(BBI, RightMBB);
9236  WorkList.push_back(
9237  {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
9238  // Put Cond in a virtual register to make it available from the new blocks.
9239  ExportFromCurrentBlock(Cond);
9240  }
9241 
9242  // Create the CaseBlock record that will be used to lower the branch.
9243  CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
9244  LeftProb, RightProb);
9245 
9246  if (W.MBB == SwitchMBB)
9247  visitSwitchCase(CB, SwitchMBB);
9248  else
9249  SwitchCases.push_back(CB);
9250 }
9251 
9252 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
9253  // Extract cases from the switch.
9255  CaseClusterVector Clusters;
9256  Clusters.reserve(SI.getNumCases());
9257  for (auto I : SI.cases()) {
9258  MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
9259  const ConstantInt *CaseVal = I.getCaseValue();
9260  BranchProbability Prob =
9261  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
9262  : BranchProbability(1, SI.getNumCases() + 1);
9263  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
9264  }
9265 
9266  MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
9267 
9268  // Cluster adjacent cases with the same destination. We do this at all
9269  // optimization levels because it's cheap to do and will make codegen faster
9270  // if there are many clusters.
9271  sortAndRangeify(Clusters);
9272 
9273  if (TM.getOptLevel() != CodeGenOpt::None) {
9274  // Replace an unreachable default with the most popular destination.
9275  // FIXME: Exploit unreachable default more aggressively.
9276  bool UnreachableDefault =
9277  isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
9278  if (UnreachableDefault && !Clusters.empty()) {
9280  unsigned MaxPop = 0;
9281  const BasicBlock *MaxBB = nullptr;
9282  for (auto I : SI.cases()) {
9283  const BasicBlock *BB = I.getCaseSuccessor();
9284  if (++Popularity[BB] > MaxPop) {
9285  MaxPop = Popularity[BB];
9286  MaxBB = BB;
9287  }
9288  }
9289  // Set new default.
9290  assert(MaxPop > 0 && MaxBB);
9291  DefaultMBB = FuncInfo.MBBMap[MaxBB];
9292 
9293  // Remove cases that were pointing to the destination that is now the
9294  // default.
9295  CaseClusterVector New;
9296  New.reserve(Clusters.size());
9297  for (CaseCluster &CC : Clusters) {
9298  if (CC.MBB != DefaultMBB)
9299  New.push_back(CC);
9300  }
9301  Clusters = std::move(New);
9302  }
9303  }
9304 
9305  // If there is only the default destination, jump there directly.
9306  MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
9307  if (Clusters.empty()) {
9308  SwitchMBB->addSuccessor(DefaultMBB);
9309  if (DefaultMBB != NextBlock(SwitchMBB)) {
9310  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
9311  getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
9312  }
9313  return;
9314  }
9315 
9316  findJumpTables(Clusters, &SI, DefaultMBB);
9317  findBitTestClusters(Clusters, &SI);
9318 
9319  DEBUG({
9320  dbgs() << "Case clusters: ";
9321  for (const CaseCluster &C : Clusters) {
9322  if (C.Kind == CC_JumpTable) dbgs() << "JT:";
9323  if (C.Kind == CC_BitTests) dbgs() << "BT:";
9324 
9325  C.Low->getValue().print(dbgs(), true);
9326  if (C.Low != C.High) {
9327  dbgs() << '-';
9328  C.High->getValue().print(dbgs(), true);
9329  }
9330  dbgs() << ' ';
9331  }
9332  dbgs() << '\n';
9333  });
9334 
9335  assert(!Clusters.empty());
9336  SwitchWorkList WorkList;
9337  CaseClusterIt First = Clusters.begin();
9338  CaseClusterIt Last = Clusters.end() - 1;
9339  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
9340  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
9341 
9342  while (!WorkList.empty()) {
9343  SwitchWorkListItem W = WorkList.back();
9344  WorkList.pop_back();
9345  unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
9346 
9347  if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
9348  !DefaultMBB->getParent()->getFunction()->optForMinSize()) {
9349  // For optimized builds, lower large range as a balanced binary tree.
9350  splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
9351  continue;
9352  }
9353 
9354  lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
9355  }
9356 }
MachineLoop * L
AttributeSet getAttributes() const
Return the parameter attributes for this call.
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
void setHasStackMap(bool s=true)
IterTy arg_end() const
Definition: CallSite.h:532
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:673
const NoneType None
Definition: None.h:23
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
void setAllowReciprocal(bool b)
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:500
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:467
BasicBlock * getSuccessor(unsigned i) const
Return a value (possibly void), from a function.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
Value * getValueOperand()
Definition: Instructions.h:391
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
std::vector< BitTestBlock > BitTestCases
BitTestCases - Vector of BitTestBlock structures used to communicate SwitchInst code generation infor...
static MVT getIntegerVT(unsigned BitWidth)
void setByValAlign(unsigned A)
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:526
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
bool isEHPad() const
Returns true if the block is a landing pad.
SelectionDAGBuilder * SDB
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
AtomicOrdering getFailureOrdering() const
Returns the ordering constraint on this cmpxchg.
Definition: Instructions.h:590
AsmDialect getDialect() const
Definition: InlineAsm.h:70
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:513
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:524
SDValue getValue(unsigned R) const
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:870
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:332
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool findValue(const Value *V) const
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, Optional< ISD::NodeType > AssertOp=None)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
This instruction extracts a struct member or array element value from an aggregate value...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
*p = old <signed v ? old : v
Definition: Instructions.h:699
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
LLVMContext * getContext() const
Definition: SelectionDAG.h:333
LLVM Argument representation.
Definition: Argument.h:34
LLVMContext & Context
static bool areJTsAllowed(const TargetLowering &TLI, const SwitchInst *SI)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:804
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
const Instruction & back() const
Definition: BasicBlock.h:242
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd)...
Definition: SelectionDAG.h:724
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:112
Unsigned minimum.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
bool hasName() const
Definition: Value.h:236
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:42
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:304
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
size_t i
bool isVolatile() const
Return true if this is a store to a volatile memory location.
Definition: Instructions.h:336
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
static F t0[256]
SynchronizationScope getSynchScope() const
Definition: Instructions.h:366
const TargetLibraryInfo * LibInfo
ArrayRef< unsigned > getIndices() const
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandExp2 - Lower an exp2 intrinsic.
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
SDDbgValue * getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool IsIndirect, uint64_t Off, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
static MVT getVectorVT(MVT VT, unsigned NumElements)
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
void setCallsUnwindInit(bool b)
bool onlyReadsMemory() const
Determine if the function does not access or only reads memory.
Definition: Function.h:321
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:39
unsigned getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
void LowerDeoptimizeCall(const CallInst *CI)
AtomicOrdering getSuccessOrdering() const
Returns the ordering constraint on this cmpxchg.
Definition: Instructions.h:585
An instruction for ordering other memory operations.
Definition: Instructions.h:430
void CopyValueToVirtualRegister(const Value *V, unsigned Reg)
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:504
const Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall...
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6...
DILocalScope * getScope() const
Get the local scope for this variable.
void addLandingPadInfo(const LandingPadInst &I, MachineBasicBlock &MBB)
Extract the exception handling information from the landingpad instruction and add them to the specif...
unsigned getNumOperands() const
Definition: User.h:167
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
Definition: Statepoint.h:300
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:329
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:699
MCSymbol * getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen...
Definition: MCContext.cpp:139
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:329
void LowerStatepoint(ImmutableStatepoint Statepoint, const BasicBlock *EHPadBB=nullptr)
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:330
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:615
This class represents a function call, abstracting a target machine's calling convention.
bool isConvergent() const
Determine if the call is convergent.
Definition: CallSite.h:478
unsigned InferPtrAlignment(SDValue Ptr) const
Infer alignment of a load / store address.
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
bool usesUnderscoreSetJmp() const
Determine if we should use _setjmp or setjmp to implement llvm.setjmp.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
const std::string & getAsmString() const
Definition: InlineAsm.h:82
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
gep_type_iterator gep_type_end(const User *GEP)
SDValue getBasicBlock(MachineBasicBlock *MBB)
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:237
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
*p = old <unsigned v ? old : v
Definition: Instructions.h:703
void setValue(const Value *V, SDValue NewN)
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:777
unsigned getID() const
Return the register class ID number.
void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
Function Alias Analysis Results
*p = old >unsigned v ? old : v
Definition: Instructions.h:701
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
unsigned getSizeInBits() const
This instruction constructs a fixed permutation of two input vectors.
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const
For some targets, an LLVM struct type must be broken down into multiple simple types, but the calling convention specifies that the entire struct must be passed in a block of consecutive registers.
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:192
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:711
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog2 - Lower a log2 intrinsic.
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.cpp:238
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:736
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:100
arg_iterator arg_end()
Definition: Function.h:559
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:830
const SDValue & getOperand(unsigned Num) const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:234
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
This class represents a sign extension of integer types.
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:664
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
An instruction for reading from memory.
Definition: Instructions.h:164
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:170
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:330
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:669
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
Hexagon Common GEP
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDDbgValue * getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t Off, const DebugLoc &DL, unsigned O)
FrameIndex.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:387
Type * getElementType() const
Definition: DerivedTypes.h:462
Same for subtraction.
Definition: ISDOpcodes.h:240
loop data prefetch
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:168
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
Definition: Attributes.cpp:994
virtual unsigned getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with strcmp
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
void setNoSignedZeros(bool b)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
*p = old >signed v ? old : v
Definition: Instructions.h:697
DebugLoc getCurDebugLoc() const
uint64_t High
uint64_t getOffset() const
static unsigned getUnderlyingArgReg(const SDValue &N)
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
bool optForSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:464
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:690
void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *, unsigned)
Set the swifterror virtual register in the SwiftErrorVRegDefMap for this basic block.
void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter)
Add a dbg_value SDNode.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:212
bool optForMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:461
SDValue getExternalSymbol(const char *Sym, EVT VT)
Signed maximum.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:165
unsigned arg_size() const
Definition: CallSite.h:211
void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI)
Determine if any floating-point values are being passed to this variadic function, and set the MachineModuleInfo's usesVAFloatArgument flag if so.
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
MachineJumpTableInfo * getOrCreateJumpTableInfo(unsigned JTEntryKind)
getOrCreateJumpTableInfo - Get the JumpTableInfo for this function, if it does already exist...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts...
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:228
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition: Type.h:239
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic...
Definition: ISDOpcodes.h:114
ArrayRef< unsigned > getIndices() const
Instruction * getFirstNonPHIOrDbg()
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:187
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void setNoSignedWrap(bool b)
lazy value info
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
Value * getAddress() const
Definition: IntrinsicInst.h:91
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
The address of a basic block.
Definition: Constants.h:822
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
bool hasOperandBundlesOtherThan(ArrayRef< uint32_t > IDs) const
Return true if this operand bundle user contains operand bundles with tags other than those specified...
Definition: InstrTypes.h:1501
void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:606
bool hasNoNaNs() const
Determine whether the no-NaNs flag is set.
bool isUnconditional() const
A description of a memory reference used in the backend.
unsigned DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
void setHasPatchPoint(bool s=true)
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:299
void AddInlineAsmOperands(unsigned Kind, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
AddInlineAsmOperands - Add this value to the specified inlineasm node operand list.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp, in cases where that is faster than a libcall...
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:250
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:648
bool noSignedZeros() const
Definition: Operator.h:196
void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
struct fuzzer::@269 Flags
void setVectorReduction(bool b)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
const HexagonInstrInfo * TII
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:143
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:440
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false)
static bool hasOnlySelectUsers(const Value *Cond)
Shift and rotation operations.
Definition: ISDOpcodes.h:344
X86_INTR - x86 hardware interrupt context.
Definition: CallingConv.h:169
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:452
Class to represent struct types.
Definition: DerivedTypes.h:199
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TW, BranchProbability FW)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
bool hasOptimizedCodeGen(LibFunc::Func F) const
Tests if the function is both available and a candidate for optimized code generation.
const Value * SwiftErrorArg
The swifterror argument of the current function.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:327
void visitJumpTable(JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
Definition: CallSite.h:102
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
bool isReg() const
isReg - Tests if this is a MO_Register operand.
unsigned getNumArgOperands() const
Return the number of call arguments.
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:180
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:475
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
std::vector< MachineBasicBlock * >::iterator succ_iterator
SmallVector< EVT, 4 > ValueVTs
ValueVTs - The value types of the values, which may not be legal, and may need be promoted or synthes...
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
iterator begin() const
begin/end - Return all of the registers in this class.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1218
Reg
All possible values of the reg field in the ModR/M byte.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
NaN behavior not applicable.
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA)...
Definition: ISDOpcodes.h:96
The memory access is dereferenceable (i.e., doesn't trap).
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:233
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:719
SynchronizationScope
Definition: Instructions.h:50
static bool isRegDefEarlyClobberKind(unsigned Flag)
Definition: InlineAsm.h:277
static unsigned convertMemFlagWordToMatchingFlagWord(unsigned InputFlag)
Definition: InlineAsm.h:319
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
Definition: ValueTypes.h:206
void setByValSize(unsigned S)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
void setFunctionContextIndex(int I)
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:589
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned getEVTAlignment(EVT MemoryVT) const
Compute the default alignment value for the given type.
SynchronizationScope getSynchScope() const
Definition: Instructions.h:245
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
SmallVector< ISD::InputArg, 32 > Ins
AtomicOrdering
Atomic ordering for LLVM's memory model.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:106
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:611
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:239
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:418
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first...
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:31
Class to represent function types.
Definition: DerivedTypes.h:102
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:873
AtomicOrdering getOrdering() const
Returns the ordering constraint on this RMW.
Definition: Instructions.h:767
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value...
#define F(x, y, z)
Definition: MD5.cpp:51
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - get or set the calling convention of the call.
Definition: CallSite.h:308
CATCHPAD - Represents a catchpad instruction.
Definition: ISDOpcodes.h:597
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:410
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool usesUnderscoreLongJmp() const
Determine if we should use _longjmp or longjmp to implement llvm.longjmp.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here...
Definition: ISDOpcodes.h:118
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
const LiveOutInfo * GetLiveOutRegInfo(unsigned Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:564
void GetReturnInfo(Type *ReturnType, AttributeSet attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:535
MachineBasicBlock * MBB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:363
const RegList & Regs
This contains information for each constraint that we are lowering.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo.ValueMap for a virtual register.
This instruction compares its operands according to the predicate given to the constructor.
static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &DL, SDISelAsmOperandInfo &OpInfo)
GetRegistersForValue - Assign registers (virtual or physical) for the specified operand.
BasicBlock * getSuccessor(unsigned i) const
SmallVector< ISD::OutputArg, 32 > Outs
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
This class represents a no-op cast from one type to another.
unsigned getCurrentCallSite()
Get the call site currently being processed, if any.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1279
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
Definition: Type.h:160
bool ShouldEmitAsBranches(const std::vector< CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI)
visitPow - Lower a pow intrinsic.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:737
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
An instruction for storing to memory.
Definition: Instructions.h:300
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
uint64_t * Cases
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:842
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
bool allowReciprocal() const
Definition: Operator.h:197
Libcall getMEMCPY_ELEMENT_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_ATOMIC - Return MEMCPY_ELEMENT_ATOMIC_* value for the given element size or UNKNOW_...
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:656
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:487
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber.
Definition: InlineAsm.h:120
static bool isRegDefKind(unsigned Flag)
Definition: InlineAsm.h:274
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements. ...
Definition: SelectionDAG.h:647
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
element_iterator elements_begin() const
Debug location.
void setStackProtectorIndex(int I)
bool noInfs() const
Definition: Operator.h:195
BasicBlock * getSuccessor() const
void setOrigAlign(unsigned A)
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
Type * getScalarType() const LLVM_READONLY
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.cpp:44
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:151
bool doesNotAccessMemory() const
Determine if the function does not access memory.
Definition: Function.h:313
Type * getElementType() const
Definition: DerivedTypes.h:336
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
bool isAtomic() const
Return true if this instruction has an AtomicOrdering of unordered or higher.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
Class to represent pointers.
Definition: DerivedTypes.h:443
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void append(const RegsForValue &RHS)
append - Add the specified values to this one.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:453
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:109
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
static const fltSemantics & IEEEsingle()
Definition: APFloat.cpp:100
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:121
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:274
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:743
bool unsafeAlgebra() const
Definition: Operator.h:198
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1695
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:254
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:74
SDNode * getNode() const
get the SDNode which holds the desired result
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
The memory access is volatile.
void update(const llvm::TargetLowering::AsmOperandInfo &OpInfo)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
Definition: Instructions.h:830
bool getLibFunc(StringRef funcName, LibFunc::Func &F) const
Searches for a particular function name.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:268
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
#define P(N)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
element_iterator elements_end() const
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:551
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:218
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr) const
getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from this value and returns the resu...
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:348
FunctionType * getFunctionType() const
Definition: CallSite.h:315
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:52
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:111
bool HasTailCall
HasTailCall - This is set to true if a call in the current block has been translated as a tail call...
unsigned getVectorNumElements() const
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:81
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type. ...
bool hasSideEffects() const
Definition: InlineAsm.h:68
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
MVT - Machine Value Type.
SynchronizationScope getSynchScope() const
Returns whether this RMW is atomic between threads or only within a single thread.
Definition: Instructions.h:773
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
Definition: InstrTypes.h:79
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
DIExpression * getExpression() const
Definition: IntrinsicInst.h:97
const Function * getParent() const
Definition: Argument.h:48
Simple binary floating point operators.
Definition: ISDOpcodes.h:246
Conditional or Unconditional Branch instruction.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:219
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align=0, bool Vol=false, bool ReadMem=true, bool WriteMem=true, unsigned Size=0)
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getMinimumJumpTableEntries() const
Return lower limit for number of blocks in a jump table.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
This function has undefined behavior.
static unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI)
Get the size in bits of Reg.
This is an important base class in LLVM.
Definition: Constant.h:42
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:637
bool isVector() const
isVector - Return true if this is a vector value type.
Resume the propagation of an exception.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
bool isWrappedSet() const
Return true if this set wraps around the top of the range.
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:279
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Indirect Branch Instruction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:109
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1952
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:335
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed...
Definition: Operator.h:128
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:154
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences ""for some float libcalls"), cl::location(LimitFloatPrecision), cl::init(0))
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:667
uint32_t Offset
void clear()
Clear the memory usage of this object.
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual void EmitFunctionEntryCode()
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
void setNoInfs(bool b)
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:880
Utility class for integer arithmetic operators which may exhibit overflow - Add, Sub, and Mul.
Definition: Operator.h:75
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1255
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, Type *LoadTy, SelectionDAGBuilder &Builder)
unsigned getOpcode() const
uint64_t getNumElements() const
Definition: DerivedTypes.h:335
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:676
Value * getOperand(unsigned i) const
Definition: User.h:145
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
Value * getPointerOperand()
Definition: Instructions.h:270
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:679
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
arg_iterator arg_begin()
Definition: Function.h:550
The memory access is non-temporal.
static cl::opt< unsigned > JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, cl::desc("Minimum density for building a jump table in ""a normal function"))
Minimum jump table density for normal functions.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
Constant Vector Declarations.
Definition: Constants.h:490
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:960
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
Definition: InlineAsm.h:311
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL)
ConstantFoldLoadFromConstPtr - Return the value that a load from C would produce if it is constant an...
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:350
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:153
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:485
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
unsigned getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:392
bool isEmptySet() const
Return true if this set contains no members.
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:276
EVT - Extended Value Type.
Definition: ValueTypes.h:31
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:213
Type * getType() const
getType - Return the type of the instruction that generated this call site
Definition: CallSite.h:258
static bool getUniformBase(const Value *&Ptr, SDValue &Base, SDValue &Index, SelectionDAGBuilder *SDB)
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array...
std::vector< ArgListEntry > ArgListTy
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:654
This structure contains all information that is necessary for lowering calls.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:213
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
SDDbgValue * getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, uint64_t Off, const DebugLoc &DL, unsigned O)
Constant.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
This class contains a discriminated union of information about pointers in memory operands...
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:391
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1118
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandExp - Lower an exp intrinsic.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:378
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
unsigned getNumSuccessors() const
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
BasicBlock * getSuccessor(unsigned i) const
TargetIntrinsicInfo - Interface to description of machine instruction set.
The memory access writes data.
virtual Value * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
BasicBlock * getUnwindDest() const
static bool isVectorReductionOp(const User *I)
Checks if the given instruction performs a vector reduction, in which case we have the freedom to alt...
bool startsWithDeref() const
Is the first element a DW_OP_deref?.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it, emit nodes to copy the value into the virtual registers.
Given one NaN input, returns the non-NaN.
IterTy arg_begin() const
Definition: CallSite.h:528
Representation for a specific memory location.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:709
CallInst * getTerminatingDeoptimizeCall()
Returns the call instruction calling .experimental.deoptimize prior to the terminating return instruc...
Definition: BasicBlock.cpp:165
virtual const TargetFrameLowering * getFrameLowering() const
std::vector< AsmOperandInfo > AsmOperandInfoVector
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:689
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:234
DIExpression * getExpression() const
Value * getValOperand()
Definition: Instructions.h:781
Iterator for intrusive lists based on ilist_node.
void setNoUnsignedWrap(bool b)
AtomicOrdering getOrdering() const
Returns the ordering effect of this store.
Definition: Instructions.h:355
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1: ...
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:494
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the generic address space (address sp...
Definition: DerivedTypes.h:458
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
InstrTy * getInstruction() const
Definition: CallSite.h:93
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:266
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition: APInt.cpp:533
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:408
const MCContext & getContext() const
ValTy * getArgument(unsigned ArgNo) const
Definition: CallSite.h:178
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
Unsigned maximum.
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:220
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
Provides information about what library functions are available for the current target.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:175
void clear()
clear - Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuil...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
This class represents a range of values.
Definition: ConstantRange.h:45
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:218
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:566
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
Floating point minnum.
An SDNode that represents everything that will be needed to construct a MachineInstr.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:625
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:347
void setCallsEHReturn(bool b)
void visit(const Instruction &I)
Value * getCatchSwitchParentPad() const
Get the parentPad of this catchret's catchpad's catchswitch.
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block, return true.
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:490
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:337
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
Definition: Instructions.h:234
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:198
static cl::opt< unsigned > OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden, cl::desc("Minimum density for building a jump table in ""an optsize function"))
Minimum jump table density for -Os or -Oz functions.
void setNoNaNs(bool b)
Represents one node in the SelectionDAG.
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog - Lower a log intrinsic.
const MachineInstrBuilder & addFrameIndex(int Idx) const
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
UpdateSplitBlock - When an MBB was split during scheduling, update the references that need to refer ...
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:506
BinOp getOperation() const
Definition: Instructions.h:724
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:584
DWARF expression.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:146
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:513
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
AttributeSet getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:176
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
SDValue getCALLSEQ_START(SDValue Chain, SDValue Op, const SDLoc &DL)
Return a new CALLSEQ_START node, which always must have a glue result (to ensure it's not CSE'd)...
Definition: SelectionDAG.h:715
SynchronizationScope getSynchScope() const
Definition: Instructions.h:468
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Class to represent vector types.
Definition: DerivedTypes.h:369
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SynchronizationScope SynchScope=CrossThread, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
static const unsigned MaxParallelChains
Class for arbitrary precision integers.
Definition: APInt.h:77
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool hasEHFunclets() const
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
void visitBitTestCase(BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
BranchProbabilityInfo * BPI
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:354
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:195
This file defines the FastISel class.
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
iterator_range< user_iterator > users()
Definition: Value.h:370
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:438
std::vector< JumpTableBlock > JTCases
JTCases - Vector of JumpTable structures used to communicate SwitchInst code generation information...
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
static AttributeSet getReturnAttrs(TargetLowering::CallLoweringInfo &CLI)
Returns an AttributeSet representing the attributes applied to the return value of the given call...
Value * getValue() const
static bool IsOnlyUsedInZeroEqualityComparison(const Value *V)
IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the value is equal or not-eq...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:130
Value * getCondition() const
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:259
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:527
bool isMinValue() const
Determine if this is the smallest unsigned value.
Definition: APInt.h:366
int getStackProtectorIndex() const
Return the index for the stack protector object.
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned char TargetFlags=0)
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:602
SmallVector< SDValue, 8 > PendingLoads
PendingLoads - Loads are not emitted to the program immediately.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
static cl::opt< bool > EnableFMFInDAG("enable-fmf-dag", cl::init(true), cl::Hidden, cl::desc("Enable fast-math-flags for DAG nodes"))
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca...
Definition: ISDOpcodes.h:758
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
CallLoweringInfo & setTailCall(bool Value=true)
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1132
bool isDereferenceablePointer(const Value *V, const DataLayout &DL, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if this is always a dereferenceable pointer.
Definition: Loads.cpp:143
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:560
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:633
static StringRef getRealLinkageName(StringRef Name)
If special LLVM prefix that is used to inform the asm printer to not emit usual symbol prefix before ...
Definition: GlobalValue.h:444
unsigned countLeadingOnes() const
Count the number of leading one bits.
Definition: APInt.cpp:676
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
cl::opt< std::string > TrapFuncName("trap-func", cl::Hidden, cl::desc("Emit a call to trap function rather than a trap instruction"), cl::init(""))
These are IR-level optimization flags that may be propagated to SDNodes.
Represents a use of a SDNode.
CallLoweringInfo & setConvergent(bool Value=true)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
SmallVector< SDValue, 32 > OutVals
Value * getCondition() const
Analysis providing branch probability information.
BasicBlock * getDefaultDest() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:142
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:247
bool isUndef() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:610
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:169
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
Definition: Instructions.h:457
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:188
This represents the llvm.dbg.value instruction.
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:227
Value * getPointerOperand()
Definition: Instructions.h:777
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:665
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:676
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value...
The memory access always returns the same value (or traps).
iterator end()
Definition: DenseMap.h:69
bool isTailCall() const
ExtraFlags(ImmutableCallSite CS)
iterator find(const KeyT &Val)
Definition: DenseMap.h:127
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
Same for multiplication.
Definition: ISDOpcodes.h:243
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling...
unsigned CreateReg(MVT VT)
CreateReg - Allocate a single virtual register for the given type.
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:391
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block...
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.setjmp intrinsic.
Definition: ISDOpcodes.h:108
unsigned getNumCases() const
Return the number of 'cases' in this switch instruction, excluding the default case.
Signed minimum.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, const AllocaInst *Alloca=nullptr)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
getCopyFromRegs - If there was virtual register allocated for the value V emit CopyFromReg of the spe...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool hasLocalLinkage() const
Definition: GlobalValue.h:415
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin...
Definition: ISDOpcodes.h:102
Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return the attribute object that exists at the given index.
void setUnsafeAlgebra(bool b)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope)
Gets a node for an atomic cmpxchg op.
bool isStatepoint(ImmutableCallSite CS)
Definition: Statepoint.cpp:27
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, const BasicBlock *EHPadBB=nullptr)
const unsigned Kind
Multiway switch.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:291
unsigned getReg() const
getReg - Returns the register number.
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:178
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:118
bool use_empty() const
Definition: Value.h:299
void erase(iterator MBBI)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
SmallVector< MVT, 4 > RegVTs
RegVTs - The value types of the registers.
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:93
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned char TargetFlags=0)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
static bool createVirtualRegs(SmallVector< unsigned, 4 > &Regs, unsigned NumRegs, MVT RegVT, SelectionDAG &DAG)
Fill Regs with NumRegs new virtual registers of type RegVT.
Given one NaN input, returns the NaN.
const BasicBlock & front() const
Definition: Function.h:542
void setUnusedArgValue(const Value *V, SDValue NewN)
unsigned getParamAlignment(unsigned i) const
Extract the alignment for a call or parameter (0=unknown).
Definition: Function.h:296
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:91
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual const TargetInstrInfo * getInstrInfo() const
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability >> &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:537
LLVM Value Representation.
Definition: Value.h:71
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:249
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1007
void init(GCFunctionInfo *gfi, AliasAnalysis &aa, const TargetLibraryInfo *li)
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:111
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
DILocalVariable * getVariable() const
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
void setDebugLoc(DebugLoc dl)
Set source location info.
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TW, BranchProbability FW)
FindMergedConditions - If Cond is an expression like.
SDValue getValueType(EVT)
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:93
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:685
static const Function * getParent(const Value *V)
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:533
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:331
Invoke instruction.
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:125
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
std::vector< CaseBlock > SwitchCases
SwitchCases - Vector of CaseBlock structures used to communicate SwitchInst code generation informati...
#define DEBUG(X)
Definition: Debug.h:100
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SDValue getControlRoot()
getControlRoot - Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports items.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:168
vt_iterator vt_begin() const
vt_begin / vt_end - Loop over all of the value types that can be represented by values in this regist...
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
Garbage collection metadata for a single function.
Definition: GCMetadata.h:74
MachineModuleInfo & getMMI() const
void clearDanglingDebugInfo()
clearDanglingDebugInfo - Clear the dangling debug information map.
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
static BranchProbability getZero()
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
LPadToCallSiteMap - Map a landing pad to the call site indexes.
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog10 - Lower a log10 intrinsic.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
MVT getVectorElementType() const
static bool isVolatile(Instruction *Inst)
Conversion operators.
Definition: ISDOpcodes.h:397
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:465
int * Ptr
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1382
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:698
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
bool isUIntN(unsigned N, uint64_t x)
isUIntN - Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:360
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:102
void setArgumentFrameIndex(const Argument *A, int FI)
setArgumentFrameIndex - Record frame index for the byval argument.
DenseMap< const Constant *, unsigned > ConstantsOut
bool isAlignStack() const
Definition: InlineAsm.h:69
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
bool isBigEndian() const
Definition: DataLayout.h:221
SynchronizationScope getSynchScope() const
Returns whether this cmpxchg is atomic between threads or only within a single thread.
Definition: Instructions.h:596
SDDbgValue - Holds the information from a dbg_value node through SDISel.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:479
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:89
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:516
virtual unsigned getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
SmallVector< unsigned, 4 > Regs
Regs - This list holds the registers assigned to the values.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.cpp:234
Value * getPointerOperand()
Definition: Instructions.h:394
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo)
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:694
static MachineOperand CreateFI(int Idx)
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests" ...
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:411
const BasicBlock * getParent() const
Definition: Instruction.h:62
bool isExportedInst(const Value *V)
isExportedInst - Return true if the specified value is an instruction exported from its block...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1107
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
static BranchProbability getBranchProbStackProtector(bool IsLikely)
FunctionLoweringInfo * FuncInfo
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
auto count_if(R &&Range, UnaryPredicate P) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:799
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:226
SmallVector< int, 16 > getShuffleMask() const
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall...
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:287
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
getCopyToRegs - Emit a series of CopyToReg nodes that copies the specified value into the registers s...
bool noNaNs() const
Flag queries.
Definition: Operator.h:194
This class contains meta information specific to a module.
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:222
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
an instruction to allocate memory on the stack
Definition: Instructions.h:60
unsigned getLiveInPhysReg(unsigned VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
This instruction inserts a struct field of array element value into an aggregate value.
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:529
unsigned InitializeRegForValue(const Value *V)
gep_type_iterator gep_type_begin(const User *GEP)
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
BRIND - Indirect branch.
Definition: ISDOpcodes.h:556
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
static bool InBlock(const Value *V, const BasicBlock *BB)
void resize(size_type N)
Definition: SmallVector.h:352
unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register in SwiftErrorVRegDefMap for this basic block...
This class is used to represent ISD::LOAD nodes.
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:545