LLVM  3.7.0
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
25 #include "llvm/CodeGen/FastISel.h"
36 #include "llvm/CodeGen/StackMaps.h"
38 #include "llvm/IR/CallingConv.h"
39 #include "llvm/IR/Constants.h"
40 #include "llvm/IR/DataLayout.h"
41 #include "llvm/IR/DebugInfo.h"
42 #include "llvm/IR/DerivedTypes.h"
43 #include "llvm/IR/Function.h"
44 #include "llvm/IR/GlobalVariable.h"
45 #include "llvm/IR/InlineAsm.h"
46 #include "llvm/IR/Instructions.h"
47 #include "llvm/IR/IntrinsicInst.h"
48 #include "llvm/IR/Intrinsics.h"
49 #include "llvm/IR/LLVMContext.h"
50 #include "llvm/IR/Module.h"
51 #include "llvm/IR/Statepoint.h"
52 #include "llvm/MC/MCSymbol.h"
54 #include "llvm/Support/Debug.h"
65 #include <algorithm>
66 using namespace llvm;
67 
68 #define DEBUG_TYPE "isel"
69 
70 /// LimitFloatPrecision - Generate low-precision inline sequences for
71 /// some float libcalls (6, 8 or 12 bits).
72 static unsigned LimitFloatPrecision;
73 
75 LimitFPPrecision("limit-float-precision",
76  cl::desc("Generate low-precision inline sequences "
77  "for some float libcalls"),
79  cl::init(0));
80 
81 static cl::opt<bool>
82 EnableFMFInDAG("enable-fmf-dag", cl::init(false), cl::Hidden,
83  cl::desc("Enable fast-math-flags for DAG nodes"));
84 
85 // Limit the width of DAG chains. This is important in general to prevent
86 // DAG-based analysis from blowing up. For example, alias analysis and
87 // load clustering may not complete in reasonable time. It is difficult to
88 // recognize and avoid this situation within each individual analysis, and
89 // future analyses are likely to have the same behavior. Limiting DAG width is
90 // the safe approach and will be especially important with global DAGs.
91 //
92 // MaxParallelChains default is arbitrarily high to avoid affecting
93 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
94 // sequence over this should have been converted to llvm.memcpy by the
95 // frontend. It easy to induce this behavior with .ll code such as:
96 // %buffer = alloca [4096 x i8]
97 // %data = load [4096 x i8]* %argPtr
98 // store [4096 x i8] %data, [4096 x i8]* %buffer
99 static const unsigned MaxParallelChains = 64;
100 
102  const SDValue *Parts, unsigned NumParts,
103  MVT PartVT, EVT ValueVT, const Value *V);
104 
105 /// getCopyFromParts - Create a value that contains the specified legal parts
106 /// combined into the value they represent. If the parts combine to a type
107 /// larger then ValueVT then AssertOp can be used to specify whether the extra
108 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
109 /// (ISD::AssertSext).
111  const SDValue *Parts,
112  unsigned NumParts, MVT PartVT, EVT ValueVT,
113  const Value *V,
114  ISD::NodeType AssertOp = ISD::DELETED_NODE) {
115  if (ValueVT.isVector())
116  return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
117  PartVT, ValueVT, V);
118 
119  assert(NumParts > 0 && "No parts to assemble!");
120  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
121  SDValue Val = Parts[0];
122 
123  if (NumParts > 1) {
124  // Assemble the value from multiple parts.
125  if (ValueVT.isInteger()) {
126  unsigned PartBits = PartVT.getSizeInBits();
127  unsigned ValueBits = ValueVT.getSizeInBits();
128 
129  // Assemble the power of 2 part.
130  unsigned RoundParts = NumParts & (NumParts - 1) ?
131  1 << Log2_32(NumParts) : NumParts;
132  unsigned RoundBits = PartBits * RoundParts;
133  EVT RoundVT = RoundBits == ValueBits ?
134  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
135  SDValue Lo, Hi;
136 
137  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
138 
139  if (RoundParts > 2) {
140  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
141  PartVT, HalfVT, V);
142  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
143  RoundParts / 2, PartVT, HalfVT, V);
144  } else {
145  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
146  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
147  }
148 
149  if (DAG.getDataLayout().isBigEndian())
150  std::swap(Lo, Hi);
151 
152  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
153 
154  if (RoundParts < NumParts) {
155  // Assemble the trailing non-power-of-2 part.
156  unsigned OddParts = NumParts - RoundParts;
157  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
158  Hi = getCopyFromParts(DAG, DL,
159  Parts + RoundParts, OddParts, PartVT, OddVT, V);
160 
161  // Combine the round and odd parts.
162  Lo = Val;
163  if (DAG.getDataLayout().isBigEndian())
164  std::swap(Lo, Hi);
165  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
166  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
167  Hi =
168  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
169  DAG.getConstant(Lo.getValueType().getSizeInBits(), DL,
170  TLI.getPointerTy(DAG.getDataLayout())));
171  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
172  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
173  }
174  } else if (PartVT.isFloatingPoint()) {
175  // FP split into multiple FP parts (for ppcf128)
176  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
177  "Unexpected split");
178  SDValue Lo, Hi;
179  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
180  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
181  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
182  std::swap(Lo, Hi);
183  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
184  } else {
185  // FP split into integer parts (soft fp)
186  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
187  !PartVT.isVector() && "Unexpected split");
188  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
189  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
190  }
191  }
192 
193  // There is now one part, held in Val. Correct it to match ValueVT.
194  EVT PartEVT = Val.getValueType();
195 
196  if (PartEVT == ValueVT)
197  return Val;
198 
199  if (PartEVT.isInteger() && ValueVT.isInteger()) {
200  if (ValueVT.bitsLT(PartEVT)) {
201  // For a truncate, see if we have any information to
202  // indicate whether the truncated bits will always be
203  // zero or sign-extension.
204  if (AssertOp != ISD::DELETED_NODE)
205  Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
206  DAG.getValueType(ValueVT));
207  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
208  }
209  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
210  }
211 
212  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
213  // FP_ROUND's are always exact here.
214  if (ValueVT.bitsLT(Val.getValueType()))
215  return DAG.getNode(
216  ISD::FP_ROUND, DL, ValueVT, Val,
217  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
218 
219  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
220  }
221 
222  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
223  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
224 
225  llvm_unreachable("Unknown mismatch!");
226 }
227 
229  const Twine &ErrMsg) {
230  const Instruction *I = dyn_cast_or_null<Instruction>(V);
231  if (!V)
232  return Ctx.emitError(ErrMsg);
233 
234  const char *AsmError = ", possible invalid constraint for vector type";
235  if (const CallInst *CI = dyn_cast<CallInst>(I))
236  if (isa<InlineAsm>(CI->getCalledValue()))
237  return Ctx.emitError(I, ErrMsg + AsmError);
238 
239  return Ctx.emitError(I, ErrMsg);
240 }
241 
242 /// getCopyFromPartsVector - Create a value that contains the specified legal
243 /// parts combined into the value they represent. If the parts combine to a
244 /// type larger then ValueVT then AssertOp can be used to specify whether the
245 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
246 /// ValueVT (ISD::AssertSext).
248  const SDValue *Parts, unsigned NumParts,
249  MVT PartVT, EVT ValueVT, const Value *V) {
250  assert(ValueVT.isVector() && "Not a vector value");
251  assert(NumParts > 0 && "No parts to assemble!");
252  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
253  SDValue Val = Parts[0];
254 
255  // Handle a multi-element vector.
256  if (NumParts > 1) {
257  EVT IntermediateVT;
258  MVT RegisterVT;
259  unsigned NumIntermediates;
260  unsigned NumRegs =
261  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
262  NumIntermediates, RegisterVT);
263  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
264  NumParts = NumRegs; // Silence a compiler warning.
265  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
266  assert(RegisterVT.getSizeInBits() ==
267  Parts[0].getSimpleValueType().getSizeInBits() &&
268  "Part type sizes don't match!");
269 
270  // Assemble the parts into intermediate operands.
271  SmallVector<SDValue, 8> Ops(NumIntermediates);
272  if (NumIntermediates == NumParts) {
273  // If the register was not expanded, truncate or copy the value,
274  // as appropriate.
275  for (unsigned i = 0; i != NumParts; ++i)
276  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
277  PartVT, IntermediateVT, V);
278  } else if (NumParts > 0) {
279  // If the intermediate type was expanded, build the intermediate
280  // operands from the parts.
281  assert(NumParts % NumIntermediates == 0 &&
282  "Must expand into a divisible number of parts!");
283  unsigned Factor = NumParts / NumIntermediates;
284  for (unsigned i = 0; i != NumIntermediates; ++i)
285  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
286  PartVT, IntermediateVT, V);
287  }
288 
289  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
290  // intermediate operands.
291  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
293  DL, ValueVT, Ops);
294  }
295 
296  // There is now one part, held in Val. Correct it to match ValueVT.
297  EVT PartEVT = Val.getValueType();
298 
299  if (PartEVT == ValueVT)
300  return Val;
301 
302  if (PartEVT.isVector()) {
303  // If the element type of the source/dest vectors are the same, but the
304  // parts vector has more elements than the value vector, then we have a
305  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
306  // elements we want.
307  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
308  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
309  "Cannot narrow, it would be a lossy transformation");
310  return DAG.getNode(
311  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
312  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
313  }
314 
315  // Vector/Vector bitcast.
316  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
317  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
318 
319  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
320  "Cannot handle this kind of promotion");
321  // Promoted vector extract
322  bool Smaller = ValueVT.bitsLE(PartEVT);
323  return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
324  DL, ValueVT, Val);
325 
326  }
327 
328  // Trivial bitcast if the types are the same size and the destination
329  // vector type is legal.
330  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
331  TLI.isTypeLegal(ValueVT))
332  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
333 
334  // Handle cases such as i8 -> <1 x i1>
335  if (ValueVT.getVectorNumElements() != 1) {
337  "non-trivial scalar-to-vector conversion");
338  return DAG.getUNDEF(ValueVT);
339  }
340 
341  if (ValueVT.getVectorNumElements() == 1 &&
342  ValueVT.getVectorElementType() != PartEVT) {
343  bool Smaller = ValueVT.bitsLE(PartEVT);
344  Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
345  DL, ValueVT.getScalarType(), Val);
346  }
347 
348  return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
349 }
350 
351 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
352  SDValue Val, SDValue *Parts, unsigned NumParts,
353  MVT PartVT, const Value *V);
354 
355 /// getCopyToParts - Create a series of nodes that contain the specified value
356 /// split into legal parts. If the parts contain more bits than Val, then, for
357 /// integers, ExtendKind can be used to specify how to generate the extra bits.
358 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
359  SDValue Val, SDValue *Parts, unsigned NumParts,
360  MVT PartVT, const Value *V,
361  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
362  EVT ValueVT = Val.getValueType();
363 
364  // Handle the vector case separately.
365  if (ValueVT.isVector())
366  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
367 
368  unsigned PartBits = PartVT.getSizeInBits();
369  unsigned OrigNumParts = NumParts;
370  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
371  "Copying to an illegal type!");
372 
373  if (NumParts == 0)
374  return;
375 
376  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
377  EVT PartEVT = PartVT;
378  if (PartEVT == ValueVT) {
379  assert(NumParts == 1 && "No-op copy with multiple parts!");
380  Parts[0] = Val;
381  return;
382  }
383 
384  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
385  // If the parts cover more bits than the value has, promote the value.
386  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
387  assert(NumParts == 1 && "Do not know what to promote to!");
388  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
389  } else {
390  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
391  ValueVT.isInteger() &&
392  "Unknown mismatch!");
393  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
394  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
395  if (PartVT == MVT::x86mmx)
396  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
397  }
398  } else if (PartBits == ValueVT.getSizeInBits()) {
399  // Different types of the same size.
400  assert(NumParts == 1 && PartEVT != ValueVT);
401  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
402  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
403  // If the parts cover less bits than value has, truncate the value.
404  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
405  ValueVT.isInteger() &&
406  "Unknown mismatch!");
407  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
408  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
409  if (PartVT == MVT::x86mmx)
410  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
411  }
412 
413  // The value may have changed - recompute ValueVT.
414  ValueVT = Val.getValueType();
415  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
416  "Failed to tile the value with PartVT!");
417 
418  if (NumParts == 1) {
419  if (PartEVT != ValueVT)
421  "scalar-to-vector conversion failed");
422 
423  Parts[0] = Val;
424  return;
425  }
426 
427  // Expand the value into multiple parts.
428  if (NumParts & (NumParts - 1)) {
429  // The number of parts is not a power of 2. Split off and copy the tail.
430  assert(PartVT.isInteger() && ValueVT.isInteger() &&
431  "Do not know what to expand to!");
432  unsigned RoundParts = 1 << Log2_32(NumParts);
433  unsigned RoundBits = RoundParts * PartBits;
434  unsigned OddParts = NumParts - RoundParts;
435  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
436  DAG.getIntPtrConstant(RoundBits, DL));
437  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
438 
439  if (DAG.getDataLayout().isBigEndian())
440  // The odd parts were reversed by getCopyToParts - unreverse them.
441  std::reverse(Parts + RoundParts, Parts + NumParts);
442 
443  NumParts = RoundParts;
444  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
445  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
446  }
447 
448  // The number of parts is a power of 2. Repeatedly bisect the value using
449  // EXTRACT_ELEMENT.
450  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
452  ValueVT.getSizeInBits()),
453  Val);
454 
455  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
456  for (unsigned i = 0; i < NumParts; i += StepSize) {
457  unsigned ThisBits = StepSize * PartBits / 2;
458  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
459  SDValue &Part0 = Parts[i];
460  SDValue &Part1 = Parts[i+StepSize/2];
461 
462  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
463  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
464  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
465  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
466 
467  if (ThisBits == PartBits && ThisVT != PartVT) {
468  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
469  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
470  }
471  }
472  }
473 
474  if (DAG.getDataLayout().isBigEndian())
475  std::reverse(Parts, Parts + OrigNumParts);
476 }
477 
478 
479 /// getCopyToPartsVector - Create a series of nodes that contain the specified
480 /// value split into legal parts.
482  SDValue Val, SDValue *Parts, unsigned NumParts,
483  MVT PartVT, const Value *V) {
484  EVT ValueVT = Val.getValueType();
485  assert(ValueVT.isVector() && "Not a vector");
486  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
487 
488  if (NumParts == 1) {
489  EVT PartEVT = PartVT;
490  if (PartEVT == ValueVT) {
491  // Nothing to do.
492  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
493  // Bitconvert vector->vector case.
494  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
495  } else if (PartVT.isVector() &&
496  PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
497  PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
498  EVT ElementVT = PartVT.getVectorElementType();
499  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
500  // undef elements.
502  for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
503  Ops.push_back(DAG.getNode(
504  ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
505  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
506 
507  for (unsigned i = ValueVT.getVectorNumElements(),
508  e = PartVT.getVectorNumElements(); i != e; ++i)
509  Ops.push_back(DAG.getUNDEF(ElementVT));
510 
511  Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
512 
513  // FIXME: Use CONCAT for 2x -> 4x.
514 
515  //SDValue UndefElts = DAG.getUNDEF(VectorTy);
516  //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
517  } else if (PartVT.isVector() &&
518  PartEVT.getVectorElementType().bitsGE(
519  ValueVT.getVectorElementType()) &&
520  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
521 
522  // Promoted vector extract
523  bool Smaller = PartEVT.bitsLE(ValueVT);
524  Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
525  DL, PartVT, Val);
526  } else{
527  // Vector -> scalar conversion.
528  assert(ValueVT.getVectorNumElements() == 1 &&
529  "Only trivial vector-to-scalar conversions should get here!");
530  Val = DAG.getNode(
531  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
532  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
533 
534  bool Smaller = ValueVT.bitsLE(PartVT);
535  Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
536  DL, PartVT, Val);
537  }
538 
539  Parts[0] = Val;
540  return;
541  }
542 
543  // Handle a multi-element vector.
544  EVT IntermediateVT;
545  MVT RegisterVT;
546  unsigned NumIntermediates;
547  unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
548  IntermediateVT,
549  NumIntermediates, RegisterVT);
550  unsigned NumElements = ValueVT.getVectorNumElements();
551 
552  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
553  NumParts = NumRegs; // Silence a compiler warning.
554  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
555 
556  // Split the vector into intermediate operands.
557  SmallVector<SDValue, 8> Ops(NumIntermediates);
558  for (unsigned i = 0; i != NumIntermediates; ++i) {
559  if (IntermediateVT.isVector())
560  Ops[i] =
561  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
562  DAG.getConstant(i * (NumElements / NumIntermediates), DL,
563  TLI.getVectorIdxTy(DAG.getDataLayout())));
564  else
565  Ops[i] = DAG.getNode(
566  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
567  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
568  }
569 
570  // Split the intermediate operands into legal parts.
571  if (NumParts == NumIntermediates) {
572  // If the register was not expanded, promote or copy the value,
573  // as appropriate.
574  for (unsigned i = 0; i != NumParts; ++i)
575  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
576  } else if (NumParts > 0) {
577  // If the intermediate type was expanded, split each the value into
578  // legal parts.
579  assert(NumIntermediates != 0 && "division by zero");
580  assert(NumParts % NumIntermediates == 0 &&
581  "Must expand into a divisible number of parts!");
582  unsigned Factor = NumParts / NumIntermediates;
583  for (unsigned i = 0; i != NumIntermediates; ++i)
584  getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
585  }
586 }
587 
589 
591  EVT valuevt)
592  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
593 
595  const DataLayout &DL, unsigned Reg, Type *Ty) {
596  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
597 
598  for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
599  EVT ValueVT = ValueVTs[Value];
600  unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
601  MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
602  for (unsigned i = 0; i != NumRegs; ++i)
603  Regs.push_back(Reg + i);
604  RegVTs.push_back(RegisterVT);
605  Reg += NumRegs;
606  }
607 }
608 
609 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
610 /// this value and returns the result as a ValueVT value. This uses
611 /// Chain/Flag as the input and updates them for the output Chain/Flag.
612 /// If the Flag pointer is NULL, no flag is used.
614  FunctionLoweringInfo &FuncInfo,
615  SDLoc dl,
616  SDValue &Chain, SDValue *Flag,
617  const Value *V) const {
618  // A Value with type {} or [0 x %t] needs no registers.
619  if (ValueVTs.empty())
620  return SDValue();
621 
622  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
623 
624  // Assemble the legal parts into the final values.
625  SmallVector<SDValue, 4> Values(ValueVTs.size());
627  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
628  // Copy the legal parts from the registers.
629  EVT ValueVT = ValueVTs[Value];
630  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
631  MVT RegisterVT = RegVTs[Value];
632 
633  Parts.resize(NumRegs);
634  for (unsigned i = 0; i != NumRegs; ++i) {
635  SDValue P;
636  if (!Flag) {
637  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
638  } else {
639  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
640  *Flag = P.getValue(2);
641  }
642 
643  Chain = P.getValue(1);
644  Parts[i] = P;
645 
646  // If the source register was virtual and if we know something about it,
647  // add an assert node.
649  !RegisterVT.isInteger() || RegisterVT.isVector())
650  continue;
651 
653  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
654  if (!LOI)
655  continue;
656 
657  unsigned RegSize = RegisterVT.getSizeInBits();
658  unsigned NumSignBits = LOI->NumSignBits;
659  unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
660 
661  if (NumZeroBits == RegSize) {
662  // The current value is a zero.
663  // Explicitly express that as it would be easier for
664  // optimizations to kick in.
665  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
666  continue;
667  }
668 
669  // FIXME: We capture more information than the dag can represent. For
670  // now, just use the tightest assertzext/assertsext possible.
671  bool isSExt = true;
672  EVT FromVT(MVT::Other);
673  if (NumSignBits == RegSize)
674  isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1
675  else if (NumZeroBits >= RegSize-1)
676  isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1
677  else if (NumSignBits > RegSize-8)
678  isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8
679  else if (NumZeroBits >= RegSize-8)
680  isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8
681  else if (NumSignBits > RegSize-16)
682  isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16
683  else if (NumZeroBits >= RegSize-16)
684  isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
685  else if (NumSignBits > RegSize-32)
686  isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32
687  else if (NumZeroBits >= RegSize-32)
688  isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
689  else
690  continue;
691 
692  // Add an assertion node.
693  assert(FromVT != MVT::Other);
694  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
695  RegisterVT, P, DAG.getValueType(FromVT));
696  }
697 
698  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
699  NumRegs, RegisterVT, ValueVT, V);
700  Part += NumRegs;
701  Parts.clear();
702  }
703 
704  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
705 }
706 
707 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
708 /// specified value into the registers specified by this object. This uses
709 /// Chain/Flag as the input and updates them for the output Chain/Flag.
710 /// If the Flag pointer is NULL, no flag is used.
712  SDValue &Chain, SDValue *Flag, const Value *V,
713  ISD::NodeType PreferredExtendType) const {
714  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
715  ISD::NodeType ExtendKind = PreferredExtendType;
716 
717  // Get the list of the values's legal parts.
718  unsigned NumRegs = Regs.size();
719  SmallVector<SDValue, 8> Parts(NumRegs);
720  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
721  EVT ValueVT = ValueVTs[Value];
722  unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
723  MVT RegisterVT = RegVTs[Value];
724 
725  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
726  ExtendKind = ISD::ZERO_EXTEND;
727 
728  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
729  &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
730  Part += NumParts;
731  }
732 
733  // Copy the parts into the registers.
734  SmallVector<SDValue, 8> Chains(NumRegs);
735  for (unsigned i = 0; i != NumRegs; ++i) {
736  SDValue Part;
737  if (!Flag) {
738  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
739  } else {
740  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
741  *Flag = Part.getValue(1);
742  }
743 
744  Chains[i] = Part.getValue(0);
745  }
746 
747  if (NumRegs == 1 || Flag)
748  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
749  // flagged to it. That is the CopyToReg nodes and the user are considered
750  // a single scheduling unit. If we create a TokenFactor and return it as
751  // chain, then the TokenFactor is both a predecessor (operand) of the
752  // user as well as a successor (the TF operands are flagged to the user).
753  // c1, f1 = CopyToReg
754  // c2, f2 = CopyToReg
755  // c3 = TokenFactor c1, c2
756  // ...
757  // = op c3, ..., f2
758  Chain = Chains[NumRegs-1];
759  else
760  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
761 }
762 
763 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
764 /// operand list. This adds the code marker and includes the number of
765 /// values added into it.
766 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
767  unsigned MatchingIdx, SDLoc dl,
768  SelectionDAG &DAG,
769  std::vector<SDValue> &Ops) const {
770  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
771 
772  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
773  if (HasMatching)
774  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
775  else if (!Regs.empty() &&
777  // Put the register class of the virtual registers in the flag word. That
778  // way, later passes can recompute register class constraints for inline
779  // assembly as well as normal instructions.
780  // Don't do this for tied operands that can use the regclass information
781  // from the def.
783  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
784  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
785  }
786 
787  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
788  Ops.push_back(Res);
789 
790  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
791  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
792  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
793  MVT RegisterVT = RegVTs[Value];
794  for (unsigned i = 0; i != NumRegs; ++i) {
795  assert(Reg < Regs.size() && "Mismatch in # registers expected");
796  unsigned TheReg = Regs[Reg++];
797  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
798 
799  if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
800  // If we clobbered the stack pointer, MFI should know about it.
801  assert(DAG.getMachineFunction().getFrameInfo()->
802  hasOpaqueSPAdjustment());
803  }
804  }
805  }
806 }
807 
809  const TargetLibraryInfo *li) {
810  AA = &aa;
811  GFI = gfi;
812  LibInfo = li;
813  DL = &DAG.getDataLayout();
814  Context = DAG.getContext();
815  LPadToCallSiteMap.clear();
816 }
817 
818 /// clear - Clear out the current SelectionDAG and the associated
819 /// state and prepare this SelectionDAGBuilder object to be used
820 /// for a new block. This doesn't clear out information about
821 /// additional blocks that are needed to complete switch lowering
822 /// or PHI node updating; that information is cleared out as it is
823 /// consumed.
825  NodeMap.clear();
826  UnusedArgNodeMap.clear();
827  PendingLoads.clear();
828  PendingExports.clear();
829  CurInst = nullptr;
830  HasTailCall = false;
831  SDNodeOrder = LowestSDNodeOrder;
833 }
834 
835 /// clearDanglingDebugInfo - Clear the dangling debug information
836 /// map. This function is separated from the clear so that debug
837 /// information that is dangling in a basic block can be properly
838 /// resolved in a different basic block. This allows the
839 /// SelectionDAG to resolve dangling debug information attached
840 /// to PHI nodes.
842  DanglingDebugInfoMap.clear();
843 }
844 
845 /// getRoot - Return the current virtual root of the Selection DAG,
846 /// flushing any PendingLoad items. This must be done before emitting
847 /// a store or any other node that may need to be ordered after any
848 /// prior load instructions.
849 ///
851  if (PendingLoads.empty())
852  return DAG.getRoot();
853 
854  if (PendingLoads.size() == 1) {
855  SDValue Root = PendingLoads[0];
856  DAG.setRoot(Root);
857  PendingLoads.clear();
858  return Root;
859  }
860 
861  // Otherwise, we have to make a token factor node.
863  PendingLoads);
864  PendingLoads.clear();
865  DAG.setRoot(Root);
866  return Root;
867 }
868 
869 /// getControlRoot - Similar to getRoot, but instead of flushing all the
870 /// PendingLoad items, flush all the PendingExports items. It is necessary
871 /// to do this before emitting a terminator instruction.
872 ///
874  SDValue Root = DAG.getRoot();
875 
876  if (PendingExports.empty())
877  return Root;
878 
879  // Turn all of the CopyToReg chains into one factored node.
880  if (Root.getOpcode() != ISD::EntryToken) {
881  unsigned i = 0, e = PendingExports.size();
882  for (; i != e; ++i) {
883  assert(PendingExports[i].getNode()->getNumOperands() > 1);
884  if (PendingExports[i].getNode()->getOperand(0) == Root)
885  break; // Don't add the root if we already indirectly depend on it.
886  }
887 
888  if (i == e)
889  PendingExports.push_back(Root);
890  }
891 
893  PendingExports);
894  PendingExports.clear();
895  DAG.setRoot(Root);
896  return Root;
897 }
898 
900  // Set up outgoing PHI node register values before emitting the terminator.
901  if (isa<TerminatorInst>(&I))
902  HandlePHINodesInSuccessorBlocks(I.getParent());
903 
904  ++SDNodeOrder;
905 
906  CurInst = &I;
907 
908  visit(I.getOpcode(), I);
909 
910  if (!isa<TerminatorInst>(&I) && !HasTailCall)
912 
913  CurInst = nullptr;
914 }
915 
916 void SelectionDAGBuilder::visitPHI(const PHINode &) {
917  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
918 }
919 
920 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
921  // Note: this doesn't use InstVisitor, because it has to work with
922  // ConstantExpr's in addition to instructions.
923  switch (Opcode) {
924  default: llvm_unreachable("Unknown instruction type encountered!");
925  // Build the switch statement using the Instruction.def file.
926 #define HANDLE_INST(NUM, OPCODE, CLASS) \
927  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
928 #include "llvm/IR/Instruction.def"
929  }
930 }
931 
932 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
933 // generate the debug data structures now that we've seen its definition.
935  SDValue Val) {
936  DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
937  if (DDI.getDI()) {
938  const DbgValueInst *DI = DDI.getDI();
939  DebugLoc dl = DDI.getdl();
940  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
941  DILocalVariable *Variable = DI->getVariable();
942  DIExpression *Expr = DI->getExpression();
943  assert(Variable->isValidLocationForIntrinsic(dl) &&
944  "Expected inlined-at fields to agree");
945  uint64_t Offset = DI->getOffset();
946  // A dbg.value for an alloca is always indirect.
947  bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
948  SDDbgValue *SDV;
949  if (Val.getNode()) {
950  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, IsIndirect,
951  Val)) {
952  SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
953  IsIndirect, Offset, dl, DbgSDNodeOrder);
954  DAG.AddDbgValue(SDV, Val.getNode(), false);
955  }
956  } else
957  DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
958  DanglingDebugInfoMap[V] = DanglingDebugInfo();
959  }
960 }
961 
962 /// getCopyFromRegs - If there was virtual register allocated for the value V
963 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
966  SDValue Result;
967 
968  if (It != FuncInfo.ValueMap.end()) {
969  unsigned InReg = It->second;
971  DAG.getDataLayout(), InReg, Ty);
972  SDValue Chain = DAG.getEntryNode();
973  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
974  resolveDanglingDebugInfo(V, Result);
975  }
976 
977  return Result;
978 }
979 
980 /// getValue - Return an SDValue for the given Value.
982  // If we already have an SDValue for this value, use it. It's important
983  // to do this first, so that we don't create a CopyFromReg if we already
984  // have a regular SDValue.
985  SDValue &N = NodeMap[V];
986  if (N.getNode()) return N;
987 
988  // If there's a virtual register allocated and initialized for this
989  // value, use it.
990  SDValue copyFromReg = getCopyFromRegs(V, V->getType());
991  if (copyFromReg.getNode()) {
992  return copyFromReg;
993  }
994 
995  // Otherwise create a new SDValue and remember it.
996  SDValue Val = getValueImpl(V);
997  NodeMap[V] = Val;
998  resolveDanglingDebugInfo(V, Val);
999  return Val;
1000 }
1001 
1002 // Return true if SDValue exists for the given Value
1004  return (NodeMap.find(V) != NodeMap.end()) ||
1006 }
1007 
1008 /// getNonRegisterValue - Return an SDValue for the given Value, but
1009 /// don't look in FuncInfo.ValueMap for a virtual register.
1011  // If we already have an SDValue for this value, use it.
1012  SDValue &N = NodeMap[V];
1013  if (N.getNode()) {
1014  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1015  // Remove the debug location from the node as the node is about to be used
1016  // in a location which may differ from the original debug location. This
1017  // is relevant to Constant and ConstantFP nodes because they can appear
1018  // as constant expressions inside PHI nodes.
1019  N->setDebugLoc(DebugLoc());
1020  }
1021  return N;
1022  }
1023 
1024  // Otherwise create a new SDValue and remember it.
1025  SDValue Val = getValueImpl(V);
1026  NodeMap[V] = Val;
1027  resolveDanglingDebugInfo(V, Val);
1028  return Val;
1029 }
1030 
1031 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1032 /// Create an SDValue for the given value.
1034  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1035 
1036  if (const Constant *C = dyn_cast<Constant>(V)) {
1037  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1038 
1039  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1040  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1041 
1042  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1043  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1044 
1045  if (isa<ConstantPointerNull>(C)) {
1046  unsigned AS = V->getType()->getPointerAddressSpace();
1047  return DAG.getConstant(0, getCurSDLoc(),
1048  TLI.getPointerTy(DAG.getDataLayout(), AS));
1049  }
1050 
1051  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1052  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1053 
1054  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1055  return DAG.getUNDEF(VT);
1056 
1057  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1058  visit(CE->getOpcode(), *CE);
1059  SDValue N1 = NodeMap[V];
1060  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1061  return N1;
1062  }
1063 
1064  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1066  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1067  OI != OE; ++OI) {
1068  SDNode *Val = getValue(*OI).getNode();
1069  // If the operand is an empty aggregate, there are no values.
1070  if (!Val) continue;
1071  // Add each leaf value from the operand to the Constants list
1072  // to form a flattened list of all the values.
1073  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1074  Constants.push_back(SDValue(Val, i));
1075  }
1076 
1077  return DAG.getMergeValues(Constants, getCurSDLoc());
1078  }
1079 
1080  if (const ConstantDataSequential *CDS =
1081  dyn_cast<ConstantDataSequential>(C)) {
1083  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1084  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1085  // Add each leaf value from the operand to the Constants list
1086  // to form a flattened list of all the values.
1087  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1088  Ops.push_back(SDValue(Val, i));
1089  }
1090 
1091  if (isa<ArrayType>(CDS->getType()))
1092  return DAG.getMergeValues(Ops, getCurSDLoc());
1093  return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
1094  VT, Ops);
1095  }
1096 
1097  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1098  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1099  "Unknown struct or array constant!");
1100 
1101  SmallVector<EVT, 4> ValueVTs;
1102  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1103  unsigned NumElts = ValueVTs.size();
1104  if (NumElts == 0)
1105  return SDValue(); // empty struct
1107  for (unsigned i = 0; i != NumElts; ++i) {
1108  EVT EltVT = ValueVTs[i];
1109  if (isa<UndefValue>(C))
1110  Constants[i] = DAG.getUNDEF(EltVT);
1111  else if (EltVT.isFloatingPoint())
1112  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1113  else
1114  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1115  }
1116 
1117  return DAG.getMergeValues(Constants, getCurSDLoc());
1118  }
1119 
1120  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1121  return DAG.getBlockAddress(BA, VT);
1122 
1123  VectorType *VecTy = cast<VectorType>(V->getType());
1124  unsigned NumElements = VecTy->getNumElements();
1125 
1126  // Now that we know the number and type of the elements, get that number of
1127  // elements into the Ops array based on what kind of constant it is.
1129  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1130  for (unsigned i = 0; i != NumElements; ++i)
1131  Ops.push_back(getValue(CV->getOperand(i)));
1132  } else {
1133  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1134  EVT EltVT =
1135  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1136 
1137  SDValue Op;
1138  if (EltVT.isFloatingPoint())
1139  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1140  else
1141  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1142  Ops.assign(NumElements, Op);
1143  }
1144 
1145  // Create a BUILD_VECTOR node.
1146  return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
1147  }
1148 
1149  // If this is a static alloca, generate it as the frameindex instead of
1150  // computation.
1151  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1153  FuncInfo.StaticAllocaMap.find(AI);
1154  if (SI != FuncInfo.StaticAllocaMap.end())
1155  return DAG.getFrameIndex(SI->second,
1156  TLI.getPointerTy(DAG.getDataLayout()));
1157  }
1158 
1159  // If this is an instruction which fast-isel has deferred, select it now.
1160  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1161  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1162  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1163  Inst->getType());
1164  SDValue Chain = DAG.getEntryNode();
1165  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1166  }
1167 
1168  llvm_unreachable("Can't get register for value!");
1169 }
1170 
1171 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1172  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1173  auto &DL = DAG.getDataLayout();
1174  SDValue Chain = getControlRoot();
1176  SmallVector<SDValue, 8> OutVals;
1177 
1178  if (!FuncInfo.CanLowerReturn) {
1179  unsigned DemoteReg = FuncInfo.DemoteRegister;
1180  const Function *F = I.getParent()->getParent();
1181 
1182  // Emit a store of the return value through the virtual register.
1183  // Leave Outs empty so that LowerReturn won't try to load return
1184  // registers the usual way.
1185  SmallVector<EVT, 1> PtrValueVTs;
1187  PtrValueVTs);
1188 
1189  SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
1190  SDValue RetOp = getValue(I.getOperand(0));
1191 
1192  SmallVector<EVT, 4> ValueVTs;
1194  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1195  unsigned NumValues = ValueVTs.size();
1196 
1197  SmallVector<SDValue, 4> Chains(NumValues);
1198  for (unsigned i = 0; i != NumValues; ++i) {
1200  RetPtr.getValueType(), RetPtr,
1201  DAG.getIntPtrConstant(Offsets[i],
1202  getCurSDLoc()));
1203  Chains[i] =
1204  DAG.getStore(Chain, getCurSDLoc(),
1205  SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1206  // FIXME: better loc info would be nice.
1207  Add, MachinePointerInfo(), false, false, 0);
1208  }
1209 
1211  MVT::Other, Chains);
1212  } else if (I.getNumOperands() != 0) {
1213  SmallVector<EVT, 4> ValueVTs;
1214  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1215  unsigned NumValues = ValueVTs.size();
1216  if (NumValues) {
1217  SDValue RetOp = getValue(I.getOperand(0));
1218 
1219  const Function *F = I.getParent()->getParent();
1220 
1221  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1222  if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1223  Attribute::SExt))
1224  ExtendKind = ISD::SIGN_EXTEND;
1225  else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1226  Attribute::ZExt))
1227  ExtendKind = ISD::ZERO_EXTEND;
1228 
1229  LLVMContext &Context = F->getContext();
1230  bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1232 
1233  for (unsigned j = 0; j != NumValues; ++j) {
1234  EVT VT = ValueVTs[j];
1235 
1236  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1237  VT = TLI.getTypeForExtArgOrReturn(Context, VT, ExtendKind);
1238 
1239  unsigned NumParts = TLI.getNumRegisters(Context, VT);
1240  MVT PartVT = TLI.getRegisterType(Context, VT);
1241  SmallVector<SDValue, 4> Parts(NumParts);
1243  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1244  &Parts[0], NumParts, PartVT, &I, ExtendKind);
1245 
1246  // 'inreg' on function refers to return value
1248  if (RetInReg)
1249  Flags.setInReg();
1250 
1251  // Propagate extension type if any
1252  if (ExtendKind == ISD::SIGN_EXTEND)
1253  Flags.setSExt();
1254  else if (ExtendKind == ISD::ZERO_EXTEND)
1255  Flags.setZExt();
1256 
1257  for (unsigned i = 0; i < NumParts; ++i) {
1258  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1259  VT, /*isfixed=*/true, 0, 0));
1260  OutVals.push_back(Parts[i]);
1261  }
1262  }
1263  }
1264  }
1265 
1266  bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1267  CallingConv::ID CallConv =
1270  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1271 
1272  // Verify that the target's LowerReturn behaved as expected.
1273  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1274  "LowerReturn didn't return a valid chain!");
1275 
1276  // Update the DAG with the new chain value resulting from return lowering.
1277  DAG.setRoot(Chain);
1278 }
1279 
1280 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1281 /// created for it, emit nodes to copy the value into the virtual
1282 /// registers.
1284  // Skip empty types
1285  if (V->getType()->isEmptyTy())
1286  return;
1287 
1289  if (VMI != FuncInfo.ValueMap.end()) {
1290  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1291  CopyValueToVirtualRegister(V, VMI->second);
1292  }
1293 }
1294 
1295 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1296 /// the current basic block, add it to ValueMap now so that we'll get a
1297 /// CopyTo/FromReg.
1299  // No need to export constants.
1300  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1301 
1302  // Already exported?
1303  if (FuncInfo.isExportedInst(V)) return;
1304 
1305  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1307 }
1308 
1310  const BasicBlock *FromBB) {
1311  // The operands of the setcc have to be in this block. We don't know
1312  // how to export them from some other block.
1313  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1314  // Can export from current BB.
1315  if (VI->getParent() == FromBB)
1316  return true;
1317 
1318  // Is already exported, noop.
1319  return FuncInfo.isExportedInst(V);
1320  }
1321 
1322  // If this is an argument, we can export it if the BB is the entry block or
1323  // if it is already exported.
1324  if (isa<Argument>(V)) {
1325  if (FromBB == &FromBB->getParent()->getEntryBlock())
1326  return true;
1327 
1328  // Otherwise, can only export this if it is already exported.
1329  return FuncInfo.isExportedInst(V);
1330  }
1331 
1332  // Otherwise, constants can always be exported.
1333  return true;
1334 }
1335 
1336 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1337 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
1338  const MachineBasicBlock *Dst) const {
1340  if (!BPI)
1341  return 0;
1342  const BasicBlock *SrcBB = Src->getBasicBlock();
1343  const BasicBlock *DstBB = Dst->getBasicBlock();
1344  return BPI->getEdgeWeight(SrcBB, DstBB);
1345 }
1346 
1347 void SelectionDAGBuilder::
1348 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
1349  uint32_t Weight /* = 0 */) {
1350  if (!Weight)
1351  Weight = getEdgeWeight(Src, Dst);
1352  Src->addSuccessor(Dst, Weight);
1353 }
1354 
1355 
1356 static bool InBlock(const Value *V, const BasicBlock *BB) {
1357  if (const Instruction *I = dyn_cast<Instruction>(V))
1358  return I->getParent() == BB;
1359  return true;
1360 }
1361 
1362 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1363 /// This function emits a branch and is used at the leaves of an OR or an
1364 /// AND operator tree.
1365 ///
1366 void
1368  MachineBasicBlock *TBB,
1369  MachineBasicBlock *FBB,
1370  MachineBasicBlock *CurBB,
1371  MachineBasicBlock *SwitchBB,
1372  uint32_t TWeight,
1373  uint32_t FWeight) {
1374  const BasicBlock *BB = CurBB->getBasicBlock();
1375 
1376  // If the leaf of the tree is a comparison, merge the condition into
1377  // the caseblock.
1378  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1379  // The operands of the cmp have to be in this block. We don't know
1380  // how to export them from some other block. If this is the first block
1381  // of the sequence, no exporting is needed.
1382  if (CurBB == SwitchBB ||
1383  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1384  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1385  ISD::CondCode Condition;
1386  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1387  Condition = getICmpCondCode(IC->getPredicate());
1388  } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
1389  Condition = getFCmpCondCode(FC->getPredicate());
1390  if (TM.Options.NoNaNsFPMath)
1391  Condition = getFCmpCodeWithoutNaN(Condition);
1392  } else {
1393  (void)Condition; // silence warning.
1394  llvm_unreachable("Unknown compare instruction");
1395  }
1396 
1397  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1398  TBB, FBB, CurBB, TWeight, FWeight);
1399  SwitchCases.push_back(CB);
1400  return;
1401  }
1402  }
1403 
1404  // Create a CaseBlock record representing this branch.
1405  CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
1406  nullptr, TBB, FBB, CurBB, TWeight, FWeight);
1407  SwitchCases.push_back(CB);
1408 }
1409 
1410 /// Scale down both weights to fit into uint32_t.
1411 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
1412  uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
1413  uint32_t Scale = (NewMax / UINT32_MAX) + 1;
1414  NewTrue = NewTrue / Scale;
1415  NewFalse = NewFalse / Scale;
1416 }
1417 
1418 /// FindMergedConditions - If Cond is an expression like
1420  MachineBasicBlock *TBB,
1421  MachineBasicBlock *FBB,
1422  MachineBasicBlock *CurBB,
1423  MachineBasicBlock *SwitchBB,
1424  unsigned Opc, uint32_t TWeight,
1425  uint32_t FWeight) {
1426  // If this node is not part of the or/and tree, emit it as a branch.
1427  const Instruction *BOp = dyn_cast<Instruction>(Cond);
1428  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1429  (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
1430  BOp->getParent() != CurBB->getBasicBlock() ||
1431  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1432  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1433  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1434  TWeight, FWeight);
1435  return;
1436  }
1437 
1438  // Create TmpBB after CurBB.
1439  MachineFunction::iterator BBI = CurBB;
1442  CurBB->getParent()->insert(++BBI, TmpBB);
1443 
1444  if (Opc == Instruction::Or) {
1445  // Codegen X | Y as:
1446  // BB1:
1447  // jmp_if_X TBB
1448  // jmp TmpBB
1449  // TmpBB:
1450  // jmp_if_Y TBB
1451  // jmp FBB
1452  //
1453 
1454  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1455  // The requirement is that
1456  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1457  // = TrueProb for original BB.
1458  // Assuming the original weights are A and B, one choice is to set BB1's
1459  // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
1460  // assumes that
1461  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1462  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1463  // TmpBB, but the math is more complicated.
1464 
1465  uint64_t NewTrueWeight = TWeight;
1466  uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
1467  ScaleWeights(NewTrueWeight, NewFalseWeight);
1468  // Emit the LHS condition.
1469  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1470  NewTrueWeight, NewFalseWeight);
1471 
1472  NewTrueWeight = TWeight;
1473  NewFalseWeight = 2 * (uint64_t)FWeight;
1474  ScaleWeights(NewTrueWeight, NewFalseWeight);
1475  // Emit the RHS condition into TmpBB.
1476  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1477  NewTrueWeight, NewFalseWeight);
1478  } else {
1479  assert(Opc == Instruction::And && "Unknown merge op!");
1480  // Codegen X & Y as:
1481  // BB1:
1482  // jmp_if_X TmpBB
1483  // jmp FBB
1484  // TmpBB:
1485  // jmp_if_Y TBB
1486  // jmp FBB
1487  //
1488  // This requires creation of TmpBB after CurBB.
1489 
1490  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1491  // The requirement is that
1492  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1493  // = FalseProb for original BB.
1494  // Assuming the original weights are A and B, one choice is to set BB1's
1495  // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
1496  // assumes that
1497  // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
1498 
1499  uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
1500  uint64_t NewFalseWeight = FWeight;
1501  ScaleWeights(NewTrueWeight, NewFalseWeight);
1502  // Emit the LHS condition.
1503  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1504  NewTrueWeight, NewFalseWeight);
1505 
1506  NewTrueWeight = 2 * (uint64_t)TWeight;
1507  NewFalseWeight = FWeight;
1508  ScaleWeights(NewTrueWeight, NewFalseWeight);
1509  // Emit the RHS condition into TmpBB.
1510  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1511  NewTrueWeight, NewFalseWeight);
1512  }
1513 }
1514 
1515 /// If the set of cases should be emitted as a series of branches, return true.
1516 /// If we should emit this as a bunch of and/or'd together conditions, return
1517 /// false.
1518 bool
1519 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1520  if (Cases.size() != 2) return true;
1521 
1522  // If this is two comparisons of the same values or'd or and'd together, they
1523  // will get folded into a single comparison, so don't emit two blocks.
1524  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1525  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1526  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1527  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1528  return false;
1529  }
1530 
1531  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1532  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1533  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1534  Cases[0].CC == Cases[1].CC &&
1535  isa<Constant>(Cases[0].CmpRHS) &&
1536  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1537  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1538  return false;
1539  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1540  return false;
1541  }
1542 
1543  return true;
1544 }
1545 
1546 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1547  MachineBasicBlock *BrMBB = FuncInfo.MBB;
1548 
1549  // Update machine-CFG edges.
1550  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1551 
1552  if (I.isUnconditional()) {
1553  // Update machine-CFG edges.
1554  BrMBB->addSuccessor(Succ0MBB);
1555 
1556  // If this is not a fall-through branch or optimizations are switched off,
1557  // emit the branch.
1558  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1561  DAG.getBasicBlock(Succ0MBB)));
1562 
1563  return;
1564  }
1565 
1566  // If this condition is one of the special cases we handle, do special stuff
1567  // now.
1568  const Value *CondVal = I.getCondition();
1569  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1570 
1571  // If this is a series of conditions that are or'd or and'd together, emit
1572  // this as a sequence of branches instead of setcc's with and/or operations.
1573  // As long as jumps are not expensive, this should improve performance.
1574  // For example, instead of something like:
1575  // cmp A, B
1576  // C = seteq
1577  // cmp D, E
1578  // F = setle
1579  // or C, F
1580  // jnz foo
1581  // Emit:
1582  // cmp A, B
1583  // je foo
1584  // cmp D, E
1585  // jle foo
1586  //
1587  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1589  BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
1590  BOp->getOpcode() == Instruction::Or)) {
1591  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1592  BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
1593  getEdgeWeight(BrMBB, Succ1MBB));
1594  // If the compares in later blocks need to use values not currently
1595  // exported from this block, export them now. This block should always
1596  // be the first entry.
1597  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
1598 
1599  // Allow some cases to be rejected.
1601  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1604  }
1605 
1606  // Emit the branch for this block.
1607  visitSwitchCase(SwitchCases[0], BrMBB);
1608  SwitchCases.erase(SwitchCases.begin());
1609  return;
1610  }
1611 
1612  // Okay, we decided not to do this, remove any inserted MBB's and clear
1613  // SwitchCases.
1614  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1615  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1616 
1617  SwitchCases.clear();
1618  }
1619  }
1620 
1621  // Create a CaseBlock record representing this branch.
1622  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1623  nullptr, Succ0MBB, Succ1MBB, BrMBB);
1624 
1625  // Use visitSwitchCase to actually insert the fast branch sequence for this
1626  // cond branch.
1627  visitSwitchCase(CB, BrMBB);
1628 }
1629 
1630 /// visitSwitchCase - Emits the necessary code to represent a single node in
1631 /// the binary search tree resulting from lowering a switch instruction.
1633  MachineBasicBlock *SwitchBB) {
1634  SDValue Cond;
1635  SDValue CondLHS = getValue(CB.CmpLHS);
1636  SDLoc dl = getCurSDLoc();
1637 
1638  // Build the setcc now.
1639  if (!CB.CmpMHS) {
1640  // Fold "(X == true)" to X and "(X == false)" to !X to
1641  // handle common cases produced by branch lowering.
1642  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1643  CB.CC == ISD::SETEQ)
1644  Cond = CondLHS;
1645  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1646  CB.CC == ISD::SETEQ) {
1647  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1648  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1649  } else
1650  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1651  } else {
1652  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
1653 
1654  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1655  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1656 
1657  SDValue CmpOp = getValue(CB.CmpMHS);
1658  EVT VT = CmpOp.getValueType();
1659 
1660  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1661  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1662  ISD::SETLE);
1663  } else {
1664  SDValue SUB = DAG.getNode(ISD::SUB, dl,
1665  VT, CmpOp, DAG.getConstant(Low, dl, VT));
1666  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1667  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1668  }
1669  }
1670 
1671  // Update successor info
1672  addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
1673  // TrueBB and FalseBB are always different unless the incoming IR is
1674  // degenerate. This only happens when running llc on weird IR.
1675  if (CB.TrueBB != CB.FalseBB)
1676  addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
1677 
1678  // If the lhs block is the next block, invert the condition so that we can
1679  // fall through to the lhs instead of the rhs block.
1680  if (CB.TrueBB == NextBlock(SwitchBB)) {
1681  std::swap(CB.TrueBB, CB.FalseBB);
1682  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1683  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1684  }
1685 
1686  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1687  MVT::Other, getControlRoot(), Cond,
1688  DAG.getBasicBlock(CB.TrueBB));
1689 
1690  // Insert the false branch. Do this even if it's a fall through branch,
1691  // this makes it easier to do DAG optimizations which require inverting
1692  // the branch condition.
1693  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1694  DAG.getBasicBlock(CB.FalseBB));
1695 
1696  DAG.setRoot(BrCond);
1697 }
1698 
1699 /// visitJumpTable - Emit JumpTable node in the current MBB
1701  // Emit the code for the jump table
1702  assert(JT.Reg != -1U && "Should lower JT Header first!");
1705  JT.Reg, PTy);
1706  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1707  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1708  MVT::Other, Index.getValue(1),
1709  Table, Index);
1710  DAG.setRoot(BrJumpTable);
1711 }
1712 
1713 /// visitJumpTableHeader - This function emits necessary code to produce index
1714 /// in the JumpTable from switch case.
1716  JumpTableHeader &JTH,
1717  MachineBasicBlock *SwitchBB) {
1718  SDLoc dl = getCurSDLoc();
1719 
1720  // Subtract the lowest switch case value from the value being switched on and
1721  // conditional branch to default mbb if the result is greater than the
1722  // difference between smallest and largest cases.
1723  SDValue SwitchOp = getValue(JTH.SValue);
1724  EVT VT = SwitchOp.getValueType();
1725  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1726  DAG.getConstant(JTH.First, dl, VT));
1727 
1728  // The SDNode we just created, which holds the value being switched on minus
1729  // the smallest case value, needs to be copied to a virtual register so it
1730  // can be used as an index into the jump table in a subsequent basic block.
1731  // This value may be smaller or larger than the target's pointer type, and
1732  // therefore require extension or truncating.
1733  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1734  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
1735 
1736  unsigned JumpTableReg =
1738  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
1739  JumpTableReg, SwitchOp);
1740  JT.Reg = JumpTableReg;
1741 
1742  // Emit the range check for the jump table, and branch to the default block
1743  // for the switch statement if the value being switched on exceeds the largest
1744  // case in the switch.
1745  SDValue CMP = DAG.getSetCC(
1747  Sub.getValueType()),
1748  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
1749 
1750  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1751  MVT::Other, CopyTo, CMP,
1752  DAG.getBasicBlock(JT.Default));
1753 
1754  // Avoid emitting unnecessary branches to the next block.
1755  if (JT.MBB != NextBlock(SwitchBB))
1756  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1757  DAG.getBasicBlock(JT.MBB));
1758 
1759  DAG.setRoot(BrCond);
1760 }
1761 
1762 /// Codegen a new tail for a stack protector check ParentMBB which has had its
1763 /// tail spliced into a stack protector check success bb.
1764 ///
1765 /// For a high level explanation of how this fits into the stack protector
1766 /// generation see the comment on the declaration of class
1767 /// StackProtectorDescriptor.
1768 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
1769  MachineBasicBlock *ParentBB) {
1770 
1771  // First create the loads to the guard/stack slot for the comparison.
1772  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1773  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1774 
1775  MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
1776  int FI = MFI->getStackProtectorIndex();
1777 
1778  const Value *IRGuard = SPD.getGuard();
1779  SDValue GuardPtr = getValue(IRGuard);
1780  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
1781 
1782  unsigned Align = DL->getPrefTypeAlignment(IRGuard->getType());
1783 
1784  SDValue Guard;
1785  SDLoc dl = getCurSDLoc();
1786 
1787  // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
1788  // guard value from the virtual register holding the value. Otherwise, emit a
1789  // volatile load to retrieve the stack guard value.
1790  unsigned GuardReg = SPD.getGuardReg();
1791 
1792  if (GuardReg && TLI.useLoadStackGuardNode())
1793  Guard = DAG.getCopyFromReg(DAG.getEntryNode(), dl, GuardReg,
1794  PtrTy);
1795  else
1796  Guard = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
1797  GuardPtr, MachinePointerInfo(IRGuard, 0),
1798  true, false, false, Align);
1799 
1800  SDValue StackSlot = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
1801  StackSlotPtr,
1803  true, false, false, Align);
1804 
1805  // Perform the comparison via a subtract/getsetcc.
1806  EVT VT = Guard.getValueType();
1807  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
1808 
1810  *DAG.getContext(),
1811  Sub.getValueType()),
1812  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
1813 
1814  // If the sub is not 0, then we know the guard/stackslot do not equal, so
1815  // branch to failure MBB.
1816  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1817  MVT::Other, StackSlot.getOperand(0),
1818  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
1819  // Otherwise branch to success MBB.
1820  SDValue Br = DAG.getNode(ISD::BR, dl,
1821  MVT::Other, BrCond,
1822  DAG.getBasicBlock(SPD.getSuccessMBB()));
1823 
1824  DAG.setRoot(Br);
1825 }
1826 
1827 /// Codegen the failure basic block for a stack protector check.
1828 ///
1829 /// A failure stack protector machine basic block consists simply of a call to
1830 /// __stack_chk_fail().
1831 ///
1832 /// For a high level explanation of how this fits into the stack protector
1833 /// generation see the comment on the declaration of class
1834 /// StackProtectorDescriptor.
1835 void
1836 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
1837  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1838  SDValue Chain =
1840  nullptr, 0, false, getCurSDLoc(), false, false).second;
1841  DAG.setRoot(Chain);
1842 }
1843 
1844 /// visitBitTestHeader - This function emits necessary code to produce value
1845 /// suitable for "bit tests"
1847  MachineBasicBlock *SwitchBB) {
1848  SDLoc dl = getCurSDLoc();
1849 
1850  // Subtract the minimum value
1851  SDValue SwitchOp = getValue(B.SValue);
1852  EVT VT = SwitchOp.getValueType();
1853  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1854  DAG.getConstant(B.First, dl, VT));
1855 
1856  // Check range
1857  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1858  SDValue RangeCmp = DAG.getSetCC(
1860  Sub.getValueType()),
1861  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
1862 
1863  // Determine the type of the test operands.
1864  bool UsePtrType = false;
1865  if (!TLI.isTypeLegal(VT))
1866  UsePtrType = true;
1867  else {
1868  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
1869  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
1870  // Switch table case range are encoded into series of masks.
1871  // Just use pointer type, it's guaranteed to fit.
1872  UsePtrType = true;
1873  break;
1874  }
1875  }
1876  if (UsePtrType) {
1877  VT = TLI.getPointerTy(DAG.getDataLayout());
1878  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
1879  }
1880 
1881  B.RegVT = VT.getSimpleVT();
1882  B.Reg = FuncInfo.CreateReg(B.RegVT);
1883  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
1884 
1885  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
1886 
1887  addSuccessorWithWeight(SwitchBB, B.Default);
1888  addSuccessorWithWeight(SwitchBB, MBB);
1889 
1890  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
1891  MVT::Other, CopyTo, RangeCmp,
1892  DAG.getBasicBlock(B.Default));
1893 
1894  // Avoid emitting unnecessary branches to the next block.
1895  if (MBB != NextBlock(SwitchBB))
1896  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
1897  DAG.getBasicBlock(MBB));
1898 
1899  DAG.setRoot(BrRange);
1900 }
1901 
1902 /// visitBitTestCase - this function produces one "bit test"
1904  MachineBasicBlock* NextMBB,
1905  uint32_t BranchWeightToNext,
1906  unsigned Reg,
1907  BitTestCase &B,
1908  MachineBasicBlock *SwitchBB) {
1909  SDLoc dl = getCurSDLoc();
1910  MVT VT = BB.RegVT;
1911  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
1912  SDValue Cmp;
1913  unsigned PopCount = countPopulation(B.Mask);
1914  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1915  if (PopCount == 1) {
1916  // Testing for a single bit; just compare the shift count with what it
1917  // would need to be to shift a 1 bit in that position.
1918  Cmp = DAG.getSetCC(
1919  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
1920  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
1921  ISD::SETEQ);
1922  } else if (PopCount == BB.Range) {
1923  // There is only one zero bit in the range, test for it directly.
1924  Cmp = DAG.getSetCC(
1925  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
1926  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
1927  ISD::SETNE);
1928  } else {
1929  // Make desired shift
1930  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
1931  DAG.getConstant(1, dl, VT), ShiftOp);
1932 
1933  // Emit bit tests and jumps
1934  SDValue AndOp = DAG.getNode(ISD::AND, dl,
1935  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
1936  Cmp = DAG.getSetCC(
1937  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
1938  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
1939  }
1940 
1941  // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
1942  addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
1943  // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
1944  addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
1945 
1946  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
1948  Cmp, DAG.getBasicBlock(B.TargetBB));
1949 
1950  // Avoid emitting unnecessary branches to the next block.
1951  if (NextMBB != NextBlock(SwitchBB))
1952  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
1953  DAG.getBasicBlock(NextMBB));
1954 
1955  DAG.setRoot(BrAnd);
1956 }
1957 
1958 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
1959  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
1960 
1961  // Retrieve successors.
1963  MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
1964 
1965  const Value *Callee(I.getCalledValue());
1966  const Function *Fn = dyn_cast<Function>(Callee);
1967  if (isa<InlineAsm>(Callee))
1968  visitInlineAsm(&I);
1969  else if (Fn && Fn->isIntrinsic()) {
1970  switch (Fn->getIntrinsicID()) {
1971  default:
1972  llvm_unreachable("Cannot invoke this intrinsic");
1973  case Intrinsic::donothing:
1974  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
1975  break;
1976  case Intrinsic::experimental_patchpoint_void:
1977  case Intrinsic::experimental_patchpoint_i64:
1978  visitPatchpoint(&I, LandingPad);
1979  break;
1980  case Intrinsic::experimental_gc_statepoint:
1981  LowerStatepoint(ImmutableStatepoint(&I), LandingPad);
1982  break;
1983  }
1984  } else
1985  LowerCallTo(&I, getValue(Callee), false, LandingPad);
1986 
1987  // If the value of the invoke is used outside of its defining block, make it
1988  // available as a virtual register.
1989  // We already took care of the exported value for the statepoint instruction
1990  // during call to the LowerStatepoint.
1991  if (!isStatepoint(I)) {
1993  }
1994 
1995  // Update successor info
1996  addSuccessorWithWeight(InvokeMBB, Return);
1997  addSuccessorWithWeight(InvokeMBB, LandingPad);
1998 
1999  // Drop into normal successor.
2002  DAG.getBasicBlock(Return)));
2003 }
2004 
2005 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2006  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2007 }
2008 
2009 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2010  assert(FuncInfo.MBB->isLandingPad() &&
2011  "Call to landingpad not in landing pad!");
2012 
2015  AddLandingPadInfo(LP, MMI, MBB);
2016 
2017  // If there aren't registers to copy the values into (e.g., during SjLj
2018  // exceptions), then don't bother to create these DAG nodes.
2019  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2020  if (TLI.getExceptionPointerRegister() == 0 &&
2021  TLI.getExceptionSelectorRegister() == 0)
2022  return;
2023 
2024  SmallVector<EVT, 2> ValueVTs;
2025  SDLoc dl = getCurSDLoc();
2026  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2027  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2028 
2029  // Get the two live-in registers as SDValues. The physregs have already been
2030  // copied into virtual registers.
2031  SDValue Ops[2];
2033  Ops[0] = DAG.getZExtOrTrunc(
2036  TLI.getPointerTy(DAG.getDataLayout())),
2037  dl, ValueVTs[0]);
2038  } else {
2039  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2040  }
2041  Ops[1] = DAG.getZExtOrTrunc(
2044  TLI.getPointerTy(DAG.getDataLayout())),
2045  dl, ValueVTs[1]);
2046 
2047  // Merge into one.
2049  DAG.getVTList(ValueVTs), Ops);
2050  setValue(&LP, Res);
2051 }
2052 
2053 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2054 #ifndef NDEBUG
2055  for (const CaseCluster &CC : Clusters)
2056  assert(CC.Low == CC.High && "Input clusters must be single-case");
2057 #endif
2058 
2059  std::sort(Clusters.begin(), Clusters.end(),
2060  [](const CaseCluster &a, const CaseCluster &b) {
2061  return a.Low->getValue().slt(b.Low->getValue());
2062  });
2063 
2064  // Merge adjacent clusters with the same destination.
2065  const unsigned N = Clusters.size();
2066  unsigned DstIndex = 0;
2067  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2068  CaseCluster &CC = Clusters[SrcIndex];
2069  const ConstantInt *CaseVal = CC.Low;
2070  MachineBasicBlock *Succ = CC.MBB;
2071 
2072  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2073  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2074  // If this case has the same successor and is a neighbour, merge it into
2075  // the previous cluster.
2076  Clusters[DstIndex - 1].High = CaseVal;
2077  Clusters[DstIndex - 1].Weight += CC.Weight;
2078  assert(Clusters[DstIndex - 1].Weight >= CC.Weight && "Weight overflow!");
2079  } else {
2080  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2081  sizeof(Clusters[SrcIndex]));
2082  }
2083  }
2084  Clusters.resize(DstIndex);
2085 }
2086 
2088  MachineBasicBlock *Last) {
2089  // Update JTCases.
2090  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2091  if (JTCases[i].first.HeaderBB == First)
2092  JTCases[i].first.HeaderBB = Last;
2093 
2094  // Update BitTestCases.
2095  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2096  if (BitTestCases[i].Parent == First)
2097  BitTestCases[i].Parent = Last;
2098 }
2099 
2100 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2101  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2102 
2103  // Update machine-CFG edges with unique successors.
2105  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2106  BasicBlock *BB = I.getSuccessor(i);
2107  bool Inserted = Done.insert(BB).second;
2108  if (!Inserted)
2109  continue;
2110 
2111  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2112  addSuccessorWithWeight(IndirectBrMBB, Succ);
2113  }
2114 
2117  getValue(I.getAddress())));
2118 }
2119 
2120 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2123 }
2124 
2125 void SelectionDAGBuilder::visitFSub(const User &I) {
2126  // -0.0 - X --> fneg
2127  Type *Ty = I.getType();
2128  if (isa<Constant>(I.getOperand(0)) &&
2130  SDValue Op2 = getValue(I.getOperand(1));
2132  Op2.getValueType(), Op2));
2133  return;
2134  }
2135 
2136  visitBinary(I, ISD::FSUB);
2137 }
2138 
2139 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2140  SDValue Op1 = getValue(I.getOperand(0));
2141  SDValue Op2 = getValue(I.getOperand(1));
2142 
2143  bool nuw = false;
2144  bool nsw = false;
2145  bool exact = false;
2146  FastMathFlags FMF;
2147 
2148  if (const OverflowingBinaryOperator *OFBinOp =
2149  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2150  nuw = OFBinOp->hasNoUnsignedWrap();
2151  nsw = OFBinOp->hasNoSignedWrap();
2152  }
2153  if (const PossiblyExactOperator *ExactOp =
2154  dyn_cast<const PossiblyExactOperator>(&I))
2155  exact = ExactOp->isExact();
2156  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2157  FMF = FPOp->getFastMathFlags();
2158 
2160  Flags.setExact(exact);
2161  Flags.setNoSignedWrap(nsw);
2162  Flags.setNoUnsignedWrap(nuw);
2163  if (EnableFMFInDAG) {
2164  Flags.setAllowReciprocal(FMF.allowReciprocal());
2165  Flags.setNoInfs(FMF.noInfs());
2166  Flags.setNoNaNs(FMF.noNaNs());
2167  Flags.setNoSignedZeros(FMF.noSignedZeros());
2168  Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2169  }
2170  SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2171  Op1, Op2, &Flags);
2172  setValue(&I, BinNodeValue);
2173 }
2174 
2175 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2176  SDValue Op1 = getValue(I.getOperand(0));
2177  SDValue Op2 = getValue(I.getOperand(1));
2178 
2180  Op2.getValueType(), DAG.getDataLayout());
2181 
2182  // Coerce the shift amount to the right type if we can.
2183  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2184  unsigned ShiftSize = ShiftTy.getSizeInBits();
2185  unsigned Op2Size = Op2.getValueType().getSizeInBits();
2186  SDLoc DL = getCurSDLoc();
2187 
2188  // If the operand is smaller than the shift count type, promote it.
2189  if (ShiftSize > Op2Size)
2190  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2191 
2192  // If the operand is larger than the shift count type but the shift
2193  // count type has enough bits to represent any shift value, truncate
2194  // it now. This is a common case and it exposes the truncate to
2195  // optimization early.
2196  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
2197  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2198  // Otherwise we'll need to temporarily settle for some other convenient
2199  // type. Type legalization will make adjustments once the shiftee is split.
2200  else
2201  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2202  }
2203 
2204  bool nuw = false;
2205  bool nsw = false;
2206  bool exact = false;
2207 
2208  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2209 
2210  if (const OverflowingBinaryOperator *OFBinOp =
2211  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2212  nuw = OFBinOp->hasNoUnsignedWrap();
2213  nsw = OFBinOp->hasNoSignedWrap();
2214  }
2215  if (const PossiblyExactOperator *ExactOp =
2216  dyn_cast<const PossiblyExactOperator>(&I))
2217  exact = ExactOp->isExact();
2218  }
2220  Flags.setExact(exact);
2221  Flags.setNoSignedWrap(nsw);
2222  Flags.setNoUnsignedWrap(nuw);
2223  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2224  &Flags);
2225  setValue(&I, Res);
2226 }
2227 
2228 void SelectionDAGBuilder::visitSDiv(const User &I) {
2229  SDValue Op1 = getValue(I.getOperand(0));
2230  SDValue Op2 = getValue(I.getOperand(1));
2231 
2233  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2234  cast<PossiblyExactOperator>(&I)->isExact());
2236  Op2, &Flags));
2237 }
2238 
2239 void SelectionDAGBuilder::visitICmp(const User &I) {
2241  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2242  predicate = IC->getPredicate();
2243  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2244  predicate = ICmpInst::Predicate(IC->getPredicate());
2245  SDValue Op1 = getValue(I.getOperand(0));
2246  SDValue Op2 = getValue(I.getOperand(1));
2247  ISD::CondCode Opcode = getICmpCondCode(predicate);
2248 
2250  I.getType());
2251  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2252 }
2253 
2254 void SelectionDAGBuilder::visitFCmp(const User &I) {
2256  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2257  predicate = FC->getPredicate();
2258  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2259  predicate = FCmpInst::Predicate(FC->getPredicate());
2260  SDValue Op1 = getValue(I.getOperand(0));
2261  SDValue Op2 = getValue(I.getOperand(1));
2262  ISD::CondCode Condition = getFCmpCondCode(predicate);
2263  if (TM.Options.NoNaNsFPMath)
2264  Condition = getFCmpCodeWithoutNaN(Condition);
2266  I.getType());
2267  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2268 }
2269 
2270 void SelectionDAGBuilder::visitSelect(const User &I) {
2271  SmallVector<EVT, 4> ValueVTs;
2273  ValueVTs);
2274  unsigned NumValues = ValueVTs.size();
2275  if (NumValues == 0) return;
2276 
2277  SmallVector<SDValue, 4> Values(NumValues);
2278  SDValue Cond = getValue(I.getOperand(0));
2279  SDValue LHSVal = getValue(I.getOperand(1));
2280  SDValue RHSVal = getValue(I.getOperand(2));
2281  auto BaseOps = {Cond};
2282  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2284 
2285  // Min/max matching is only viable if all output VTs are the same.
2286  if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2287  Value *LHS, *RHS;
2288  SelectPatternFlavor SPF = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2290  switch (SPF) {
2291  case SPF_UMAX: Opc = ISD::UMAX; break;
2292  case SPF_UMIN: Opc = ISD::UMIN; break;
2293  case SPF_SMAX: Opc = ISD::SMAX; break;
2294  case SPF_SMIN: Opc = ISD::SMIN; break;
2295  default: break;
2296  }
2297 
2298  EVT VT = ValueVTs[0];
2299  LLVMContext &Ctx = *DAG.getContext();
2300  auto &TLI = DAG.getTargetLoweringInfo();
2301  while (TLI.getTypeAction(Ctx, VT) == TargetLoweringBase::TypeSplitVector)
2302  VT = TLI.getTypeToTransformTo(Ctx, VT);
2303 
2304  if (Opc != ISD::DELETED_NODE && TLI.isOperationLegalOrCustom(Opc, VT) &&
2305  // If the underlying comparison instruction is used by any other instruction,
2306  // the consumed instructions won't be destroyed, so it is not profitable
2307  // to convert to a min/max.
2308  cast<SelectInst>(&I)->getCondition()->hasOneUse()) {
2309  OpCode = Opc;
2310  LHSVal = getValue(LHS);
2311  RHSVal = getValue(RHS);
2312  BaseOps = {};
2313  }
2314  }
2315 
2316  for (unsigned i = 0; i != NumValues; ++i) {
2317  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2318  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2319  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2320  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2321  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2322  Ops);
2323  }
2324 
2326  DAG.getVTList(ValueVTs), Values));
2327 }
2328 
2329 void SelectionDAGBuilder::visitTrunc(const User &I) {
2330  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2331  SDValue N = getValue(I.getOperand(0));
2333  I.getType());
2334  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2335 }
2336 
2337 void SelectionDAGBuilder::visitZExt(const User &I) {
2338  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2339  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2340  SDValue N = getValue(I.getOperand(0));
2342  I.getType());
2343  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2344 }
2345 
2346 void SelectionDAGBuilder::visitSExt(const User &I) {
2347  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2348  // SExt also can't be a cast to bool for same reason. So, nothing much to do
2349  SDValue N = getValue(I.getOperand(0));
2351  I.getType());
2352  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2353 }
2354 
2355 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2356  // FPTrunc is never a no-op cast, no need to check
2357  SDValue N = getValue(I.getOperand(0));
2358  SDLoc dl = getCurSDLoc();
2359  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2360  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2361  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2363  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2364 }
2365 
2366 void SelectionDAGBuilder::visitFPExt(const User &I) {
2367  // FPExt is never a no-op cast, no need to check
2368  SDValue N = getValue(I.getOperand(0));
2370  I.getType());
2371  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2372 }
2373 
2374 void SelectionDAGBuilder::visitFPToUI(const User &I) {
2375  // FPToUI is never a no-op cast, no need to check
2376  SDValue N = getValue(I.getOperand(0));
2378  I.getType());
2379  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2380 }
2381 
2382 void SelectionDAGBuilder::visitFPToSI(const User &I) {
2383  // FPToSI is never a no-op cast, no need to check
2384  SDValue N = getValue(I.getOperand(0));
2386  I.getType());
2387  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2388 }
2389 
2390 void SelectionDAGBuilder::visitUIToFP(const User &I) {
2391  // UIToFP is never a no-op cast, no need to check
2392  SDValue N = getValue(I.getOperand(0));
2394  I.getType());
2395  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2396 }
2397 
2398 void SelectionDAGBuilder::visitSIToFP(const User &I) {
2399  // SIToFP is never a no-op cast, no need to check
2400  SDValue N = getValue(I.getOperand(0));
2402  I.getType());
2403  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2404 }
2405 
2406 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2407  // What to do depends on the size of the integer and the size of the pointer.
2408  // We can either truncate, zero extend, or no-op, accordingly.
2409  SDValue N = getValue(I.getOperand(0));
2411  I.getType());
2412  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2413 }
2414 
2415 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2416  // What to do depends on the size of the integer and the size of the pointer.
2417  // We can either truncate, zero extend, or no-op, accordingly.
2418  SDValue N = getValue(I.getOperand(0));
2420  I.getType());
2421  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2422 }
2423 
2424 void SelectionDAGBuilder::visitBitCast(const User &I) {
2425  SDValue N = getValue(I.getOperand(0));
2426  SDLoc dl = getCurSDLoc();
2428  I.getType());
2429 
2430  // BitCast assures us that source and destination are the same size so this is
2431  // either a BITCAST or a no-op.
2432  if (DestVT != N.getValueType())
2433  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2434  DestVT, N)); // convert types.
2435  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2436  // might fold any kind of constant expression to an integer constant and that
2437  // is not what we are looking for. Only regcognize a bitcast of a genuine
2438  // constant integer as an opaque constant.
2439  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2440  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2441  /*isOpaque*/true));
2442  else
2443  setValue(&I, N); // noop cast.
2444 }
2445 
2446 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2447  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2448  const Value *SV = I.getOperand(0);
2449  SDValue N = getValue(SV);
2450  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2451 
2452  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
2453  unsigned DestAS = I.getType()->getPointerAddressSpace();
2454 
2455  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
2456  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
2457 
2458  setValue(&I, N);
2459 }
2460 
2461 void SelectionDAGBuilder::visitInsertElement(const User &I) {
2462  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2463  SDValue InVec = getValue(I.getOperand(0));
2464  SDValue InVal = getValue(I.getOperand(1));
2468  TLI.getValueType(DAG.getDataLayout(), I.getType()),
2469  InVec, InVal, InIdx));
2470 }
2471 
2472 void SelectionDAGBuilder::visitExtractElement(const User &I) {
2473  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2474  SDValue InVec = getValue(I.getOperand(0));
2478  TLI.getValueType(DAG.getDataLayout(), I.getType()),
2479  InVec, InIdx));
2480 }
2481 
2482 // Utility for visitShuffleVector - Return true if every element in Mask,
2483 // beginning from position Pos and ending in Pos+Size, falls within the
2484 // specified sequential range [L, L+Pos). or is undef.
2486  unsigned Pos, unsigned Size, int Low) {
2487  for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
2488  if (Mask[i] >= 0 && Mask[i] != Low)
2489  return false;
2490  return true;
2491 }
2492 
2493 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
2494  SDValue Src1 = getValue(I.getOperand(0));
2495  SDValue Src2 = getValue(I.getOperand(1));
2496 
2497  SmallVector<int, 8> Mask;
2498  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
2499  unsigned MaskNumElts = Mask.size();
2500 
2501  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2502  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2503  EVT SrcVT = Src1.getValueType();
2504  unsigned SrcNumElts = SrcVT.getVectorNumElements();
2505 
2506  if (SrcNumElts == MaskNumElts) {
2507  setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
2508  &Mask[0]));
2509  return;
2510  }
2511 
2512  // Normalize the shuffle vector since mask and vector length don't match.
2513  if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
2514  // Mask is longer than the source vectors and is a multiple of the source
2515  // vectors. We can use concatenate vector to make the mask and vectors
2516  // lengths match.
2517  if (SrcNumElts*2 == MaskNumElts) {
2518  // First check for Src1 in low and Src2 in high
2519  if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
2520  isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
2521  // The shuffle is concatenating two vectors together.
2523  VT, Src1, Src2));
2524  return;
2525  }
2526  // Then check for Src2 in low and Src1 in high
2527  if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
2528  isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
2529  // The shuffle is concatenating two vectors together.
2531  VT, Src2, Src1));
2532  return;
2533  }
2534  }
2535 
2536  // Pad both vectors with undefs to make them the same length as the mask.
2537  unsigned NumConcat = MaskNumElts / SrcNumElts;
2538  bool Src1U = Src1.getOpcode() == ISD::UNDEF;
2539  bool Src2U = Src2.getOpcode() == ISD::UNDEF;
2540  SDValue UndefVal = DAG.getUNDEF(SrcVT);
2541 
2542  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
2543  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
2544  MOps1[0] = Src1;
2545  MOps2[0] = Src2;
2546 
2547  Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
2548  getCurSDLoc(), VT, MOps1);
2549  Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
2550  getCurSDLoc(), VT, MOps2);
2551 
2552  // Readjust mask for new input vector length.
2553  SmallVector<int, 8> MappedOps;
2554  for (unsigned i = 0; i != MaskNumElts; ++i) {
2555  int Idx = Mask[i];
2556  if (Idx >= (int)SrcNumElts)
2557  Idx -= SrcNumElts - MaskNumElts;
2558  MappedOps.push_back(Idx);
2559  }
2560 
2561  setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
2562  &MappedOps[0]));
2563  return;
2564  }
2565 
2566  if (SrcNumElts > MaskNumElts) {
2567  // Analyze the access pattern of the vector to see if we can extract
2568  // two subvectors and do the shuffle. The analysis is done by calculating
2569  // the range of elements the mask access on both vectors.
2570  int MinRange[2] = { static_cast<int>(SrcNumElts),
2571  static_cast<int>(SrcNumElts)};
2572  int MaxRange[2] = {-1, -1};
2573 
2574  for (unsigned i = 0; i != MaskNumElts; ++i) {
2575  int Idx = Mask[i];
2576  unsigned Input = 0;
2577  if (Idx < 0)
2578  continue;
2579 
2580  if (Idx >= (int)SrcNumElts) {
2581  Input = 1;
2582  Idx -= SrcNumElts;
2583  }
2584  if (Idx > MaxRange[Input])
2585  MaxRange[Input] = Idx;
2586  if (Idx < MinRange[Input])
2587  MinRange[Input] = Idx;
2588  }
2589 
2590  // Check if the access is smaller than the vector size and can we find
2591  // a reasonable extract index.
2592  int RangeUse[2] = { -1, -1 }; // 0 = Unused, 1 = Extract, -1 = Can not
2593  // Extract.
2594  int StartIdx[2]; // StartIdx to extract from
2595  for (unsigned Input = 0; Input < 2; ++Input) {
2596  if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
2597  RangeUse[Input] = 0; // Unused
2598  StartIdx[Input] = 0;
2599  continue;
2600  }
2601 
2602  // Find a good start index that is a multiple of the mask length. Then
2603  // see if the rest of the elements are in range.
2604  StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
2605  if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
2606  StartIdx[Input] + MaskNumElts <= SrcNumElts)
2607  RangeUse[Input] = 1; // Extract from a multiple of the mask length.
2608  }
2609 
2610  if (RangeUse[0] == 0 && RangeUse[1] == 0) {
2611  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
2612  return;
2613  }
2614  if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
2615  // Extract appropriate subvector and generate a vector shuffle
2616  for (unsigned Input = 0; Input < 2; ++Input) {
2617  SDValue &Src = Input == 0 ? Src1 : Src2;
2618  if (RangeUse[Input] == 0)
2619  Src = DAG.getUNDEF(VT);
2620  else {
2621  SDLoc dl = getCurSDLoc();
2622  Src = DAG.getNode(
2623  ISD::EXTRACT_SUBVECTOR, dl, VT, Src,
2624  DAG.getConstant(StartIdx[Input], dl,
2625  TLI.getVectorIdxTy(DAG.getDataLayout())));
2626  }
2627  }
2628 
2629  // Calculate new mask.
2630  SmallVector<int, 8> MappedOps;
2631  for (unsigned i = 0; i != MaskNumElts; ++i) {
2632  int Idx = Mask[i];
2633  if (Idx >= 0) {
2634  if (Idx < (int)SrcNumElts)
2635  Idx -= StartIdx[0];
2636  else
2637  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
2638  }
2639  MappedOps.push_back(Idx);
2640  }
2641 
2642  setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
2643  &MappedOps[0]));
2644  return;
2645  }
2646  }
2647 
2648  // We can't use either concat vectors or extract subvectors so fall back to
2649  // replacing the shuffle with extract and build vector.
2650  // to insert and build vector.
2651  EVT EltVT = VT.getVectorElementType();
2652  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
2653  SDLoc dl = getCurSDLoc();
2655  for (unsigned i = 0; i != MaskNumElts; ++i) {
2656  int Idx = Mask[i];
2657  SDValue Res;
2658 
2659  if (Idx < 0) {
2660  Res = DAG.getUNDEF(EltVT);
2661  } else {
2662  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
2663  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
2664 
2666  EltVT, Src, DAG.getConstant(Idx, dl, IdxVT));
2667  }
2668 
2669  Ops.push_back(Res);
2670  }
2671 
2672  setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops));
2673 }
2674 
2675 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
2676  const Value *Op0 = I.getOperand(0);
2677  const Value *Op1 = I.getOperand(1);
2678  Type *AggTy = I.getType();
2679  Type *ValTy = Op1->getType();
2680  bool IntoUndef = isa<UndefValue>(Op0);
2681  bool FromUndef = isa<UndefValue>(Op1);
2682 
2683  unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
2684 
2685  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2686  SmallVector<EVT, 4> AggValueVTs;
2687  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
2688  SmallVector<EVT, 4> ValValueVTs;
2689  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
2690 
2691  unsigned NumAggValues = AggValueVTs.size();
2692  unsigned NumValValues = ValValueVTs.size();
2693  SmallVector<SDValue, 4> Values(NumAggValues);
2694 
2695  // Ignore an insertvalue that produces an empty object
2696  if (!NumAggValues) {
2698  return;
2699  }
2700 
2701  SDValue Agg = getValue(Op0);
2702  unsigned i = 0;
2703  // Copy the beginning value(s) from the original aggregate.
2704  for (; i != LinearIndex; ++i)
2705  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
2706  SDValue(Agg.getNode(), Agg.getResNo() + i);
2707  // Copy values from the inserted value(s).
2708  if (NumValValues) {
2709  SDValue Val = getValue(Op1);
2710  for (; i != LinearIndex + NumValValues; ++i)
2711  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
2712  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
2713  }
2714  // Copy remaining value(s) from the original aggregate.
2715  for (; i != NumAggValues; ++i)
2716  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
2717  SDValue(Agg.getNode(), Agg.getResNo() + i);
2718 
2720  DAG.getVTList(AggValueVTs), Values));
2721 }
2722 
2723 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
2724  const Value *Op0 = I.getOperand(0);
2725  Type *AggTy = Op0->getType();
2726  Type *ValTy = I.getType();
2727  bool OutOfUndef = isa<UndefValue>(Op0);
2728 
2729  unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
2730 
2731  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2732  SmallVector<EVT, 4> ValValueVTs;
2733  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
2734 
2735  unsigned NumValValues = ValValueVTs.size();
2736 
2737  // Ignore a extractvalue that produces an empty object
2738  if (!NumValValues) {
2740  return;
2741  }
2742 
2743  SmallVector<SDValue, 4> Values(NumValValues);
2744 
2745  SDValue Agg = getValue(Op0);
2746  // Copy out the selected value(s).
2747  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
2748  Values[i - LinearIndex] =
2749  OutOfUndef ?
2750  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
2751  SDValue(Agg.getNode(), Agg.getResNo() + i);
2752 
2754  DAG.getVTList(ValValueVTs), Values));
2755 }
2756 
2757 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
2758  Value *Op0 = I.getOperand(0);
2759  // Note that the pointer operand may be a vector of pointers. Take the scalar
2760  // element which holds a pointer.
2761  Type *Ty = Op0->getType()->getScalarType();
2762  unsigned AS = Ty->getPointerAddressSpace();
2763  SDValue N = getValue(Op0);
2764  SDLoc dl = getCurSDLoc();
2765 
2766  // Normalize Vector GEP - all scalar operands should be converted to the
2767  // splat vector.
2768  unsigned VectorWidth = I.getType()->isVectorTy() ?
2769  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
2770 
2771  if (VectorWidth && !N.getValueType().isVector()) {
2772  MVT VT = MVT::getVectorVT(N.getValueType().getSimpleVT(), VectorWidth);
2773  SmallVector<SDValue, 16> Ops(VectorWidth, N);
2774  N = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
2775  }
2776  for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
2777  OI != E; ++OI) {
2778  const Value *Idx = *OI;
2779  if (StructType *StTy = dyn_cast<StructType>(Ty)) {
2780  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
2781  if (Field) {
2782  // N = N + Offset
2783  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
2784  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
2785  DAG.getConstant(Offset, dl, N.getValueType()));
2786  }
2787 
2788  Ty = StTy->getElementType(Field);
2789  } else {
2790  Ty = cast<SequentialType>(Ty)->getElementType();
2791  MVT PtrTy =
2793  unsigned PtrSize = PtrTy.getSizeInBits();
2794  APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty));
2795 
2796  // If this is a scalar constant or a splat vector of constants,
2797  // handle it quickly.
2798  const auto *CI = dyn_cast<ConstantInt>(Idx);
2799  if (!CI && isa<ConstantDataVector>(Idx) &&
2800  cast<ConstantDataVector>(Idx)->getSplatValue())
2801  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
2802 
2803  if (CI) {
2804  if (CI->isZero())
2805  continue;
2806  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
2807  SDValue OffsVal = VectorWidth ?
2808  DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) :
2809  DAG.getConstant(Offs, dl, PtrTy);
2810  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal);
2811  continue;
2812  }
2813 
2814  // N = N + Idx * ElementSize;
2815  SDValue IdxN = getValue(Idx);
2816 
2817  if (!IdxN.getValueType().isVector() && VectorWidth) {
2818  MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
2819  SmallVector<SDValue, 16> Ops(VectorWidth, IdxN);
2820  IdxN = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
2821  }
2822  // If the index is smaller or larger than intptr_t, truncate or extend
2823  // it.
2824  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
2825 
2826  // If this is a multiply by a power of two, turn it into a shl
2827  // immediately. This is a very common case.
2828  if (ElementSize != 1) {
2829  if (ElementSize.isPowerOf2()) {
2830  unsigned Amt = ElementSize.logBase2();
2831  IdxN = DAG.getNode(ISD::SHL, dl,
2832  N.getValueType(), IdxN,
2833  DAG.getConstant(Amt, dl, IdxN.getValueType()));
2834  } else {
2835  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
2836  IdxN = DAG.getNode(ISD::MUL, dl,
2837  N.getValueType(), IdxN, Scale);
2838  }
2839  }
2840 
2841  N = DAG.getNode(ISD::ADD, dl,
2842  N.getValueType(), N, IdxN);
2843  }
2844  }
2845 
2846  setValue(&I, N);
2847 }
2848 
2849 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
2850  // If this is a fixed sized alloca in the entry block of the function,
2851  // allocate it statically on the stack.
2852  if (FuncInfo.StaticAllocaMap.count(&I))
2853  return; // getValue will auto-populate this.
2854 
2855  SDLoc dl = getCurSDLoc();
2856  Type *Ty = I.getAllocatedType();
2857  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2858  auto &DL = DAG.getDataLayout();
2859  uint64_t TySize = DL.getTypeAllocSize(Ty);
2860  unsigned Align =
2861  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
2862 
2863  SDValue AllocSize = getValue(I.getArraySize());
2864 
2865  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
2866  if (AllocSize.getValueType() != IntPtr)
2867  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
2868 
2869  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
2870  AllocSize,
2871  DAG.getConstant(TySize, dl, IntPtr));
2872 
2873  // Handle alignment. If the requested alignment is less than or equal to
2874  // the stack alignment, ignore it. If the size is greater than or equal to
2875  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
2876  unsigned StackAlign =
2878  if (Align <= StackAlign)
2879  Align = 0;
2880 
2881  // Round the size of the allocation up to the stack alignment size
2882  // by add SA-1 to the size.
2883  AllocSize = DAG.getNode(ISD::ADD, dl,
2884  AllocSize.getValueType(), AllocSize,
2885  DAG.getIntPtrConstant(StackAlign - 1, dl));
2886 
2887  // Mask out the low bits for alignment purposes.
2888  AllocSize = DAG.getNode(ISD::AND, dl,
2889  AllocSize.getValueType(), AllocSize,
2890  DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
2891  dl));
2892 
2893  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
2894  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
2895  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
2896  setValue(&I, DSA);
2897  DAG.setRoot(DSA.getValue(1));
2898 
2900 }
2901 
2902 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
2903  if (I.isAtomic())
2904  return visitAtomicLoad(I);
2905 
2906  const Value *SV = I.getOperand(0);
2907  SDValue Ptr = getValue(SV);
2908 
2909  Type *Ty = I.getType();
2910 
2911  bool isVolatile = I.isVolatile();
2912  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2913 
2914  // The IR notion of invariant_load only guarantees that all *non-faulting*
2915  // invariant loads result in the same value. The MI notion of invariant load
2916  // guarantees that the load can be legally moved to any location within its
2917  // containing function. The MI notion of invariant_load is stronger than the
2918  // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load
2919  // with a guarantee that the location being loaded from is dereferenceable
2920  // throughout the function's lifetime.
2921 
2922  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr &&
2924  unsigned Alignment = I.getAlignment();
2925 
2926  AAMDNodes AAInfo;
2927  I.getAAMetadata(AAInfo);
2929 
2930  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2931  SmallVector<EVT, 4> ValueVTs;
2933  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
2934  unsigned NumValues = ValueVTs.size();
2935  if (NumValues == 0)
2936  return;
2937 
2938  SDValue Root;
2939  bool ConstantMemory = false;
2940  if (isVolatile || NumValues > MaxParallelChains)
2941  // Serialize volatile loads with other side effects.
2942  Root = getRoot();
2943  else if (AA->pointsToConstantMemory(
2944  MemoryLocation(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
2945  // Do not serialize (non-volatile) loads of constant memory with anything.
2946  Root = DAG.getEntryNode();
2947  ConstantMemory = true;
2948  } else {
2949  // Do not serialize non-volatile loads against each other.
2950  Root = DAG.getRoot();
2951  }
2952 
2953  SDLoc dl = getCurSDLoc();
2954 
2955  if (isVolatile)
2956  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
2957 
2958  SmallVector<SDValue, 4> Values(NumValues);
2960  EVT PtrVT = Ptr.getValueType();
2961  unsigned ChainI = 0;
2962  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
2963  // Serializing loads here may result in excessive register pressure, and
2964  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
2965  // could recover a bit by hoisting nodes upward in the chain by recognizing
2966  // they are side-effect free or do not alias. The optimizer should really
2967  // avoid this case by converting large object/array copies to llvm.memcpy
2968  // (MaxParallelChains should always remain as failsafe).
2969  if (ChainI == MaxParallelChains) {
2970  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
2972  makeArrayRef(Chains.data(), ChainI));
2973  Root = Chain;
2974  ChainI = 0;
2975  }
2976  SDValue A = DAG.getNode(ISD::ADD, dl,
2977  PtrVT, Ptr,
2978  DAG.getConstant(Offsets[i], dl, PtrVT));
2979  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
2980  A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
2981  isNonTemporal, isInvariant, Alignment, AAInfo,
2982  Ranges);
2983 
2984  Values[i] = L;
2985  Chains[ChainI] = L.getValue(1);
2986  }
2987 
2988  if (!ConstantMemory) {
2990  makeArrayRef(Chains.data(), ChainI));
2991  if (isVolatile)
2992  DAG.setRoot(Chain);
2993  else
2994  PendingLoads.push_back(Chain);
2995  }
2996 
2998  DAG.getVTList(ValueVTs), Values));
2999 }
3000 
3001 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3002  if (I.isAtomic())
3003  return visitAtomicStore(I);
3004 
3005  const Value *SrcV = I.getOperand(0);
3006  const Value *PtrV = I.getOperand(1);
3007 
3008  SmallVector<EVT, 4> ValueVTs;
3011  SrcV->getType(), ValueVTs, &Offsets);
3012  unsigned NumValues = ValueVTs.size();
3013  if (NumValues == 0)
3014  return;
3015 
3016  // Get the lowered operands. Note that we do this after
3017  // checking if NumResults is zero, because with zero results
3018  // the operands won't have values in the map.
3019  SDValue Src = getValue(SrcV);
3020  SDValue Ptr = getValue(PtrV);
3021 
3022  SDValue Root = getRoot();
3024  EVT PtrVT = Ptr.getValueType();
3025  bool isVolatile = I.isVolatile();
3026  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3027  unsigned Alignment = I.getAlignment();
3028  SDLoc dl = getCurSDLoc();
3029 
3030  AAMDNodes AAInfo;
3031  I.getAAMetadata(AAInfo);
3032 
3033  unsigned ChainI = 0;
3034  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3035  // See visitLoad comments.
3036  if (ChainI == MaxParallelChains) {
3038  makeArrayRef(Chains.data(), ChainI));
3039  Root = Chain;
3040  ChainI = 0;
3041  }
3042  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3043  DAG.getConstant(Offsets[i], dl, PtrVT));
3044  SDValue St = DAG.getStore(Root, dl,
3045  SDValue(Src.getNode(), Src.getResNo() + i),
3046  Add, MachinePointerInfo(PtrV, Offsets[i]),
3047  isVolatile, isNonTemporal, Alignment, AAInfo);
3048  Chains[ChainI] = St;
3049  }
3050 
3051  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3052  makeArrayRef(Chains.data(), ChainI));
3053  DAG.setRoot(StoreNode);
3054 }
3055 
3056 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
3057  SDLoc sdl = getCurSDLoc();
3058 
3059  // llvm.masked.store.*(Src0, Ptr, alignemt, Mask)
3060  Value *PtrOperand = I.getArgOperand(1);
3061  SDValue Ptr = getValue(PtrOperand);
3062  SDValue Src0 = getValue(I.getArgOperand(0));
3063  SDValue Mask = getValue(I.getArgOperand(3));
3064  EVT VT = Src0.getValueType();
3065  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3066  if (!Alignment)
3067  Alignment = DAG.getEVTAlignment(VT);
3068 
3069  AAMDNodes AAInfo;
3070  I.getAAMetadata(AAInfo);
3071 
3072  MachineMemOperand *MMO =
3074  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3075  MachineMemOperand::MOStore, VT.getStoreSize(),
3076  Alignment, AAInfo);
3077  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3078  MMO, false);
3079  DAG.setRoot(StoreNode);
3080  setValue(&I, StoreNode);
3081 }
3082 
3083 // Gather/scatter receive a vector of pointers.
3084 // This vector of pointers may be represented as a base pointer + vector of
3085 // indices, it depends on GEP and instruction preceeding GEP
3086 // that calculates indices
3087 static bool getUniformBase(Value *& Ptr, SDValue& Base, SDValue& Index,
3088  SelectionDAGBuilder* SDB) {
3089 
3090  assert (Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3092  if (!Gep || Gep->getNumOperands() > 2)
3093  return false;
3094  ShuffleVectorInst *ShuffleInst =
3095  dyn_cast<ShuffleVectorInst>(Gep->getPointerOperand());
3096  if (!ShuffleInst || !ShuffleInst->getMask()->isNullValue() ||
3097  cast<Instruction>(ShuffleInst->getOperand(0))->getOpcode() !=
3098  Instruction::InsertElement)
3099  return false;
3100 
3101  Ptr = cast<InsertElementInst>(ShuffleInst->getOperand(0))->getOperand(1);
3102 
3103  SelectionDAG& DAG = SDB->DAG;
3104  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3105  // Check is the Ptr is inside current basic block
3106  // If not, look for the shuffle instruction
3107  if (SDB->findValue(Ptr))
3108  Base = SDB->getValue(Ptr);
3109  else if (SDB->findValue(ShuffleInst)) {
3110  SDValue ShuffleNode = SDB->getValue(ShuffleInst);
3111  SDLoc sdl = ShuffleNode;
3112  Base = DAG.getNode(
3114  ShuffleNode.getValueType().getScalarType(), ShuffleNode,
3115  DAG.getConstant(0, sdl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3116  SDB->setValue(Ptr, Base);
3117  }
3118  else
3119  return false;
3120 
3121  Value *IndexVal = Gep->getOperand(1);
3122  if (SDB->findValue(IndexVal)) {
3123  Index = SDB->getValue(IndexVal);
3124 
3125  if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3126  IndexVal = Sext->getOperand(0);
3127  if (SDB->findValue(IndexVal))
3128  Index = SDB->getValue(IndexVal);
3129  }
3130  return true;
3131  }
3132  return false;
3133 }
3134 
3135 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3136  SDLoc sdl = getCurSDLoc();
3137 
3138  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3139  Value *Ptr = I.getArgOperand(1);
3140  SDValue Src0 = getValue(I.getArgOperand(0));
3141  SDValue Mask = getValue(I.getArgOperand(3));
3142  EVT VT = Src0.getValueType();
3143  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3144  if (!Alignment)
3145  Alignment = DAG.getEVTAlignment(VT);
3146  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3147 
3148  AAMDNodes AAInfo;
3149  I.getAAMetadata(AAInfo);
3150 
3151  SDValue Base;
3152  SDValue Index;
3153  Value *BasePtr = Ptr;
3154  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3155 
3156  Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3158  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3159  MachineMemOperand::MOStore, VT.getStoreSize(),
3160  Alignment, AAInfo);
3161  if (!UniformBase) {
3162  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3163  Index = getValue(Ptr);
3164  }
3165  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3166  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3167  Ops, MMO);
3168  DAG.setRoot(Scatter);
3169  setValue(&I, Scatter);
3170 }
3171 
3172 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
3173  SDLoc sdl = getCurSDLoc();
3174 
3175  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3176  Value *PtrOperand = I.getArgOperand(0);
3177  SDValue Ptr = getValue(PtrOperand);
3178  SDValue Src0 = getValue(I.getArgOperand(3));
3179  SDValue Mask = getValue(I.getArgOperand(2));
3180 
3181  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3182  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3183  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3184  if (!Alignment)
3185  Alignment = DAG.getEVTAlignment(VT);
3186 
3187  AAMDNodes AAInfo;
3188  I.getAAMetadata(AAInfo);
3189  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3190 
3191  SDValue InChain = DAG.getRoot();
3193  PtrOperand, AA->getTypeStoreSize(I.getType()), AAInfo))) {
3194  // Do not serialize (non-volatile) loads of constant memory with anything.
3195  InChain = DAG.getEntryNode();
3196  }
3197 
3198  MachineMemOperand *MMO =
3200  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3202  Alignment, AAInfo, Ranges);
3203 
3204  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3206  SDValue OutChain = Load.getValue(1);
3207  DAG.setRoot(OutChain);
3208  setValue(&I, Load);
3209 }
3210 
3211 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3212  SDLoc sdl = getCurSDLoc();
3213 
3214  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3215  Value *Ptr = I.getArgOperand(0);
3216  SDValue Src0 = getValue(I.getArgOperand(3));
3217  SDValue Mask = getValue(I.getArgOperand(2));
3218 
3219  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3220  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3221  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3222  if (!Alignment)
3223  Alignment = DAG.getEVTAlignment(VT);
3224 
3225  AAMDNodes AAInfo;
3226  I.getAAMetadata(AAInfo);
3227  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3228 
3229  SDValue Root = DAG.getRoot();
3230  SDValue Base;
3231  SDValue Index;
3232  Value *BasePtr = Ptr;
3233  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3234  bool ConstantMemory = false;
3235  if (UniformBase &&
3237  MemoryLocation(BasePtr, AA->getTypeStoreSize(I.getType()), AAInfo))) {
3238  // Do not serialize (non-volatile) loads of constant memory with anything.
3239  Root = DAG.getEntryNode();
3240  ConstantMemory = true;
3241  }
3242 
3243  MachineMemOperand *MMO =
3245  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3247  Alignment, AAInfo, Ranges);
3248 
3249  if (!UniformBase) {
3250  Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3251  Index = getValue(Ptr);
3252  }
3253  SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3254  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3255  Ops, MMO);
3256 
3257  SDValue OutChain = Gather.getValue(1);
3258  if (!ConstantMemory)
3259  PendingLoads.push_back(OutChain);
3260  setValue(&I, Gather);
3261 }
3262 
3263 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3264  SDLoc dl = getCurSDLoc();
3265  AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3266  AtomicOrdering FailureOrder = I.getFailureOrdering();
3267  SynchronizationScope Scope = I.getSynchScope();
3268 
3269  SDValue InChain = getRoot();
3270 
3271  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3272  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3274  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3277  /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
3278 
3279  SDValue OutChain = L.getValue(2);
3280 
3281  setValue(&I, L);
3282  DAG.setRoot(OutChain);
3283 }
3284 
3285 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
3286  SDLoc dl = getCurSDLoc();
3287  ISD::NodeType NT;
3288  switch (I.getOperation()) {
3289  default: llvm_unreachable("Unknown atomicrmw operation");
3290  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
3291  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
3292  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
3293  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
3294  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
3295  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
3296  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
3297  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
3298  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
3299  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
3300  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
3301  }
3302  AtomicOrdering Order = I.getOrdering();
3303  SynchronizationScope Scope = I.getSynchScope();
3304 
3305  SDValue InChain = getRoot();
3306 
3307  SDValue L =
3308  DAG.getAtomic(NT, dl,
3309  getValue(I.getValOperand()).getSimpleValueType(),
3310  InChain,
3312  getValue(I.getValOperand()),
3313  I.getPointerOperand(),
3314  /* Alignment=*/ 0, Order, Scope);
3315 
3316  SDValue OutChain = L.getValue(1);
3317 
3318  setValue(&I, L);
3319  DAG.setRoot(OutChain);
3320 }
3321 
3322 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
3323  SDLoc dl = getCurSDLoc();
3324  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3325  SDValue Ops[3];
3326  Ops[0] = getRoot();
3327  Ops[1] = DAG.getConstant(I.getOrdering(), dl,
3328  TLI.getPointerTy(DAG.getDataLayout()));
3329  Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
3330  TLI.getPointerTy(DAG.getDataLayout()));
3332 }
3333 
3334 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
3335  SDLoc dl = getCurSDLoc();
3336  AtomicOrdering Order = I.getOrdering();
3337  SynchronizationScope Scope = I.getSynchScope();
3338 
3339  SDValue InChain = getRoot();
3340 
3341  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3342  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3343 
3344  if (I.getAlignment() < VT.getSizeInBits() / 8)
3345  report_fatal_error("Cannot generate unaligned atomic load");
3346 
3347  MachineMemOperand *MMO =
3349  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3352  VT.getStoreSize(),
3353  I.getAlignment() ? I.getAlignment() :
3354  DAG.getEVTAlignment(VT));
3355 
3356  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
3357  SDValue L =
3358  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3359  getValue(I.getPointerOperand()), MMO,
3360  Order, Scope);
3361 
3362  SDValue OutChain = L.getValue(1);
3363 
3364  setValue(&I, L);
3365  DAG.setRoot(OutChain);
3366 }
3367 
3368 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
3369  SDLoc dl = getCurSDLoc();
3370 
3371  AtomicOrdering Order = I.getOrdering();
3372  SynchronizationScope Scope = I.getSynchScope();
3373 
3374  SDValue InChain = getRoot();
3375 
3376  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3377  EVT VT =
3379 
3380  if (I.getAlignment() < VT.getSizeInBits() / 8)
3381  report_fatal_error("Cannot generate unaligned atomic store");
3382 
3383  SDValue OutChain =
3385  InChain,
3389  Order, Scope);
3390 
3391  DAG.setRoot(OutChain);
3392 }
3393 
3394 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
3395 /// node.
3396 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
3397  unsigned Intrinsic) {
3398  bool HasChain = !I.doesNotAccessMemory();
3399  bool OnlyLoad = HasChain && I.onlyReadsMemory();
3400 
3401  // Build the operand list.
3403  if (HasChain) { // If this intrinsic has side-effects, chainify it.
3404  if (OnlyLoad) {
3405  // We don't need to serialize loads against other loads.
3406  Ops.push_back(DAG.getRoot());
3407  } else {
3408  Ops.push_back(getRoot());
3409  }
3410  }
3411 
3412  // Info is set by getTgtMemInstrinsic
3413  TargetLowering::IntrinsicInfo Info;
3414  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3415  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
3416 
3417  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
3418  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
3419  Info.opc == ISD::INTRINSIC_W_CHAIN)
3420  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
3421  TLI.getPointerTy(DAG.getDataLayout())));
3422 
3423  // Add all operands of the call to the operand list.
3424  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
3425  SDValue Op = getValue(I.getArgOperand(i));
3426  Ops.push_back(Op);
3427  }
3428 
3429  SmallVector<EVT, 4> ValueVTs;
3430  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
3431 
3432  if (HasChain)
3433  ValueVTs.push_back(MVT::Other);
3434 
3435  SDVTList VTs = DAG.getVTList(ValueVTs);
3436 
3437  // Create the node.
3438  SDValue Result;
3439  if (IsTgtIntrinsic) {
3440  // This is target intrinsic that touches memory
3441  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
3442  VTs, Ops, Info.memVT,
3443  MachinePointerInfo(Info.ptrVal, Info.offset),
3444  Info.align, Info.vol,
3445  Info.readMem, Info.writeMem, Info.size);
3446  } else if (!HasChain) {
3447  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
3448  } else if (!I.getType()->isVoidTy()) {
3449  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
3450  } else {
3451  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
3452  }
3453 
3454  if (HasChain) {
3455  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
3456  if (OnlyLoad)
3457  PendingLoads.push_back(Chain);
3458  else
3459  DAG.setRoot(Chain);
3460  }
3461 
3462  if (!I.getType()->isVoidTy()) {
3463  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
3464  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
3465  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
3466  }
3467 
3468  setValue(&I, Result);
3469  }
3470 }
3471 
3472 /// GetSignificand - Get the significand and build it into a floating-point
3473 /// number with exponent of 1:
3474 ///
3475 /// Op = (Op & 0x007fffff) | 0x3f800000;
3476 ///
3477 /// where Op is the hexadecimal representation of floating point value.
3478 static SDValue
3480  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
3481  DAG.getConstant(0x007fffff, dl, MVT::i32));
3482  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
3483  DAG.getConstant(0x3f800000, dl, MVT::i32));
3484  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
3485 }
3486 
3487 /// GetExponent - Get the exponent:
3488 ///
3489 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
3490 ///
3491 /// where Op is the hexadecimal representation of floating point value.
3492 static SDValue
3494  SDLoc dl) {
3495  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
3496  DAG.getConstant(0x7f800000, dl, MVT::i32));
3497  SDValue t1 = DAG.getNode(
3498  ISD::SRL, dl, MVT::i32, t0,
3499  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
3500  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
3501  DAG.getConstant(127, dl, MVT::i32));
3502  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
3503 }
3504 
3505 /// getF32Constant - Get 32-bit floating point constant.
3506 static SDValue
3507 getF32Constant(SelectionDAG &DAG, unsigned Flt, SDLoc dl) {
3508  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), dl,
3509  MVT::f32);
3510 }
3511 
3513  SelectionDAG &DAG) {
3514  // IntegerPartOfX = ((int32_t)(t0);
3515  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
3516 
3517  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
3518  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
3519  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
3520 
3521  // IntegerPartOfX <<= 23;
3522  IntegerPartOfX = DAG.getNode(
3523  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
3525  DAG.getDataLayout())));
3526 
3527  SDValue TwoToFractionalPartOfX;
3528  if (LimitFloatPrecision <= 6) {
3529  // For floating-point precision of 6:
3530  //
3531  // TwoToFractionalPartOfX =
3532  // 0.997535578f +
3533  // (0.735607626f + 0.252464424f * x) * x;
3534  //
3535  // error 0.0144103317, which is 6 bits
3536  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3537  getF32Constant(DAG, 0x3e814304, dl));
3538  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
3539  getF32Constant(DAG, 0x3f3c50c8, dl));
3540  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3541  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3542  getF32Constant(DAG, 0x3f7f5e7e, dl));
3543  } else if (LimitFloatPrecision <= 12) {
3544  // For floating-point precision of 12:
3545  //
3546  // TwoToFractionalPartOfX =
3547  // 0.999892986f +
3548  // (0.696457318f +
3549  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
3550  //
3551  // error 0.000107046256, which is 13 to 14 bits
3552  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3553  getF32Constant(DAG, 0x3da235e3, dl));
3554  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
3555  getF32Constant(DAG, 0x3e65b8f3, dl));
3556  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3557  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3558  getF32Constant(DAG, 0x3f324b07, dl));
3559  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3560  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
3561  getF32Constant(DAG, 0x3f7ff8fd, dl));
3562  } else { // LimitFloatPrecision <= 18
3563  // For floating-point precision of 18:
3564  //
3565  // TwoToFractionalPartOfX =
3566  // 0.999999982f +
3567  // (0.693148872f +
3568  // (0.240227044f +
3569  // (0.554906021e-1f +
3570  // (0.961591928e-2f +
3571  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
3572  // error 2.47208000*10^(-7), which is better than 18 bits
3573  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3574  getF32Constant(DAG, 0x3924b03e, dl));
3575  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
3576  getF32Constant(DAG, 0x3ab24b87, dl));
3577  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3578  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3579  getF32Constant(DAG, 0x3c1d8c17, dl));
3580  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3581  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
3582  getF32Constant(DAG, 0x3d634a1d, dl));
3583  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
3584  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
3585  getF32Constant(DAG, 0x3e75fe14, dl));
3586  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
3587  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
3588  getF32Constant(DAG, 0x3f317234, dl));
3589  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
3590  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
3591  getF32Constant(DAG, 0x3f800000, dl));
3592  }
3593 
3594  // Add the exponent into the result in integer domain.
3595  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
3596  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
3597  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
3598 }
3599 
3600 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
3601 /// limited-precision mode.
3603  const TargetLowering &TLI) {
3604  if (Op.getValueType() == MVT::f32 &&
3606 
3607  // Put the exponent in the right bit position for later addition to the
3608  // final result:
3609  //
3610  // #define LOG2OFe 1.4426950f
3611  // t0 = Op * LOG2OFe
3612  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
3613  getF32Constant(DAG, 0x3fb8aa3b, dl));
3614  return getLimitedPrecisionExp2(t0, dl, DAG);
3615  }
3616 
3617  // No special expansion.
3618  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
3619 }
3620 
3621 /// expandLog - Lower a log intrinsic. Handles the special sequences for
3622 /// limited-precision mode.
3624  const TargetLowering &TLI) {
3625  if (Op.getValueType() == MVT::f32 &&
3627  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
3628 
3629  // Scale the exponent by log(2) [0.69314718f].
3630  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
3631  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
3632  getF32Constant(DAG, 0x3f317218, dl));
3633 
3634  // Get the significand and build it into a floating-point number with
3635  // exponent of 1.
3636  SDValue X = GetSignificand(DAG, Op1, dl);
3637 
3638  SDValue LogOfMantissa;
3639  if (LimitFloatPrecision <= 6) {
3640  // For floating-point precision of 6:
3641  //
3642  // LogofMantissa =
3643  // -1.1609546f +
3644  // (1.4034025f - 0.23903021f * x) * x;
3645  //
3646  // error 0.0034276066, which is better than 8 bits
3647  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3648  getF32Constant(DAG, 0xbe74c456, dl));
3649  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3650  getF32Constant(DAG, 0x3fb3a2b1, dl));
3651  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3652  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3653  getF32Constant(DAG, 0x3f949a29, dl));
3654  } else if (LimitFloatPrecision <= 12) {
3655  // For floating-point precision of 12:
3656  //
3657  // LogOfMantissa =
3658  // -1.7417939f +
3659  // (2.8212026f +
3660  // (-1.4699568f +
3661  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
3662  //
3663  // error 0.000061011436, which is 14 bits
3664  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3665  getF32Constant(DAG, 0xbd67b6d6, dl));
3666  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3667  getF32Constant(DAG, 0x3ee4f4b8, dl));
3668  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3669  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3670  getF32Constant(DAG, 0x3fbc278b, dl));
3671  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3672  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3673  getF32Constant(DAG, 0x40348e95, dl));
3674  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3675  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
3676  getF32Constant(DAG, 0x3fdef31a, dl));
3677  } else { // LimitFloatPrecision <= 18
3678  // For floating-point precision of 18:
3679  //
3680  // LogOfMantissa =
3681  // -2.1072184f +
3682  // (4.2372794f +
3683  // (-3.7029485f +
3684  // (2.2781945f +
3685  // (-0.87823314f +
3686  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
3687  //
3688  // error 0.0000023660568, which is better than 18 bits
3689  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3690  getF32Constant(DAG, 0xbc91e5ac, dl));
3691  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3692  getF32Constant(DAG, 0x3e4350aa, dl));
3693  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3694  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3695  getF32Constant(DAG, 0x3f60d3e3, dl));
3696  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3697  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3698  getF32Constant(DAG, 0x4011cdf0, dl));
3699  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3700  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
3701  getF32Constant(DAG, 0x406cfd1c, dl));
3702  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
3703  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
3704  getF32Constant(DAG, 0x408797cb, dl));
3705  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
3706  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
3707  getF32Constant(DAG, 0x4006dcab, dl));
3708  }
3709 
3710  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
3711  }
3712 
3713  // No special expansion.
3714  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
3715 }
3716 
3717 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
3718 /// limited-precision mode.
3720  const TargetLowering &TLI) {
3721  if (Op.getValueType() == MVT::f32 &&
3723  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
3724 
3725  // Get the exponent.
3726  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
3727 
3728  // Get the significand and build it into a floating-point number with
3729  // exponent of 1.
3730  SDValue X = GetSignificand(DAG, Op1, dl);
3731 
3732  // Different possible minimax approximations of significand in
3733  // floating-point for various degrees of accuracy over [1,2].
3734  SDValue Log2ofMantissa;
3735  if (LimitFloatPrecision <= 6) {
3736  // For floating-point precision of 6:
3737  //
3738  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
3739  //
3740  // error 0.0049451742, which is more than 7 bits
3741  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3742  getF32Constant(DAG, 0xbeb08fe0, dl));
3743  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3744  getF32Constant(DAG, 0x40019463, dl));
3745  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3746  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3747  getF32Constant(DAG, 0x3fd6633d, dl));
3748  } else if (LimitFloatPrecision <= 12) {
3749  // For floating-point precision of 12:
3750  //
3751  // Log2ofMantissa =
3752  // -2.51285454f +
3753  // (4.07009056f +
3754  // (-2.12067489f +
3755  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
3756  //
3757  // error 0.0000876136000, which is better than 13 bits
3758  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3759  getF32Constant(DAG, 0xbda7262e, dl));
3760  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3761  getF32Constant(DAG, 0x3f25280b, dl));
3762  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3763  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3764  getF32Constant(DAG, 0x4007b923, dl));
3765  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3766  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3767  getF32Constant(DAG, 0x40823e2f, dl));
3768  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3769  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
3770  getF32Constant(DAG, 0x4020d29c, dl));
3771  } else { // LimitFloatPrecision <= 18
3772  // For floating-point precision of 18:
3773  //
3774  // Log2ofMantissa =
3775  // -3.0400495f +
3776  // (6.1129976f +
3777  // (-5.3420409f +
3778  // (3.2865683f +
3779  // (-1.2669343f +
3780  // (0.27515199f -
3781  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
3782  //
3783  // error 0.0000018516, which is better than 18 bits
3784  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3785  getF32Constant(DAG, 0xbcd2769e, dl));
3786  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3787  getF32Constant(DAG, 0x3e8ce0b9, dl));
3788  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3789  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3790  getF32Constant(DAG, 0x3fa22ae7, dl));
3791  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3792  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
3793  getF32Constant(DAG, 0x40525723, dl));
3794  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3795  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
3796  getF32Constant(DAG, 0x40aaf200, dl));
3797  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
3798  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
3799  getF32Constant(DAG, 0x40c39dad, dl));
3800  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
3801  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
3802  getF32Constant(DAG, 0x4042902c, dl));
3803  }
3804 
3805  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
3806  }
3807 
3808  // No special expansion.
3809  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
3810 }
3811 
3812 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
3813 /// limited-precision mode.
3815  const TargetLowering &TLI) {
3816  if (Op.getValueType() == MVT::f32 &&
3818  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
3819 
3820  // Scale the exponent by log10(2) [0.30102999f].
3821  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
3822  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
3823  getF32Constant(DAG, 0x3e9a209a, dl));
3824 
3825  // Get the significand and build it into a floating-point number with
3826  // exponent of 1.
3827  SDValue X = GetSignificand(DAG, Op1, dl);
3828 
3829  SDValue Log10ofMantissa;
3830  if (LimitFloatPrecision <= 6) {
3831  // For floating-point precision of 6:
3832  //
3833  // Log10ofMantissa =
3834  // -0.50419619f +
3835  // (0.60948995f - 0.10380950f * x) * x;
3836  //
3837  // error 0.0014886165, which is 6 bits
3838  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3839  getF32Constant(DAG, 0xbdd49a13, dl));
3840  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
3841  getF32Constant(DAG, 0x3f1c0789, dl));
3842  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3843  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
3844  getF32Constant(DAG, 0x3f011300, dl));
3845  } else if (LimitFloatPrecision <= 12) {
3846  // For floating-point precision of 12:
3847  //
3848  // Log10ofMantissa =
3849  // -0.64831180f +
3850  // (0.91751397f +
3851  // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
3852  //
3853  // error 0.00019228036, which is better than 12 bits
3854  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3855  getF32Constant(DAG, 0x3d431f31, dl));
3856  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
3857  getF32Constant(DAG, 0x3ea21fb2, dl));
3858  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3859  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
3860  getF32Constant(DAG, 0x3f6ae232, dl));
3861  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3862  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
3863  getF32Constant(DAG, 0x3f25f7c3, dl));
3864  } else { // LimitFloatPrecision <= 18
3865  // For floating-point precision of 18:
3866  //
3867  // Log10ofMantissa =
3868  // -0.84299375f +
3869  // (1.5327582f +
3870  // (-1.0688956f +
3871  // (0.49102474f +
3872  // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
3873  //
3874  // error 0.0000037995730, which is better than 18 bits
3875  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
3876  getF32Constant(DAG, 0x3c5d51ce, dl));
3877  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
3878  getF32Constant(DAG, 0x3e00685a, dl));
3879  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
3880  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
3881  getF32Constant(DAG, 0x3efb6798, dl));
3882  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
3883  SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
3884  getF32Constant(DAG, 0x3f88d192, dl));
3885  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
3886  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
3887  getF32Constant(DAG, 0x3fc4316c, dl));
3888  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
3889  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
3890  getF32Constant(DAG, 0x3f57ce70, dl));
3891  }
3892 
3893  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
3894  }
3895 
3896  // No special expansion.
3897  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
3898 }
3899 
3900 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
3901 /// limited-precision mode.
3903  const TargetLowering &TLI) {
3904  if (Op.getValueType() == MVT::f32 &&
3906  return getLimitedPrecisionExp2(Op, dl, DAG);
3907 
3908  // No special expansion.
3909  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
3910 }
3911 
3912 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
3913 /// limited-precision mode with x == 10.0f.
3914 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
3915  SelectionDAG &DAG, const TargetLowering &TLI) {
3916  bool IsExp10 = false;
3917  if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
3919  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
3920  APFloat Ten(10.0f);
3921  IsExp10 = LHSC->isExactlyValue(Ten);
3922  }
3923  }
3924 
3925  if (IsExp10) {
3926  // Put the exponent in the right bit position for later addition to the
3927  // final result:
3928  //
3929  // #define LOG2OF10 3.3219281f
3930  // t0 = Op * LOG2OF10;
3931  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
3932  getF32Constant(DAG, 0x40549a78, dl));
3933  return getLimitedPrecisionExp2(t0, dl, DAG);
3934  }
3935 
3936  // No special expansion.
3937  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
3938 }
3939 
3940 
3941 /// ExpandPowI - Expand a llvm.powi intrinsic.
3943  SelectionDAG &DAG) {
3944  // If RHS is a constant, we can expand this out to a multiplication tree,
3945  // otherwise we end up lowering to a call to __powidf2 (for example). When
3946  // optimizing for size, we only want to do this if the expansion would produce
3947  // a small number of multiplies, otherwise we do the full expansion.
3948  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
3949  // Get the exponent as a positive value.
3950  unsigned Val = RHSC->getSExtValue();
3951  if ((int)Val < 0) Val = -Val;
3952 
3953  // powi(x, 0) -> 1.0
3954  if (Val == 0)
3955  return DAG.getConstantFP(1.0, DL, LHS.getValueType());
3956 
3957  const Function *F = DAG.getMachineFunction().getFunction();
3959  // If optimizing for size, don't insert too many multiplies. This
3960  // inserts up to 5 multiplies.
3961  countPopulation(Val) + Log2_32(Val) < 7) {
3962  // We use the simple binary decomposition method to generate the multiply
3963  // sequence. There are more optimal ways to do this (for example,
3964  // powi(x,15) generates one more multiply than it should), but this has
3965  // the benefit of being both really simple and much better than a libcall.
3966  SDValue Res; // Logically starts equal to 1.0
3967  SDValue CurSquare = LHS;
3968  while (Val) {
3969  if (Val & 1) {
3970  if (Res.getNode())
3971  Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
3972  else
3973  Res = CurSquare; // 1.0*CurSquare.
3974  }
3975 
3976  CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
3977  CurSquare, CurSquare);
3978  Val >>= 1;
3979  }
3980 
3981  // If the original was negative, invert the result, producing 1/(x*x*x).
3982  if (RHSC->getSExtValue() < 0)
3983  Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
3984  DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
3985  return Res;
3986  }
3987  }
3988 
3989  // Otherwise, expand to a libcall.
3990  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
3991 }
3992 
3993 // getTruncatedArgReg - Find underlying register used for an truncated
3994 // argument.
3995 static unsigned getTruncatedArgReg(const SDValue &N) {
3996  if (N.getOpcode() != ISD::TRUNCATE)
3997  return 0;
3998 
3999  const SDValue &Ext = N.getOperand(0);
4000  if (Ext.getOpcode() == ISD::AssertZext ||
4001  Ext.getOpcode() == ISD::AssertSext) {
4002  const SDValue &CFR = Ext.getOperand(0);
4003  if (CFR.getOpcode() == ISD::CopyFromReg)
4004  return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
4005  if (CFR.getOpcode() == ISD::TRUNCATE)
4006  return getTruncatedArgReg(CFR);
4007  }
4008  return 0;
4009 }
4010 
4011 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4012 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
4013 /// At the end of instruction selection, they will be inserted to the entry BB.
4014 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4015  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4016  DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
4017  const Argument *Arg = dyn_cast<Argument>(V);
4018  if (!Arg)
4019  return false;
4020 
4023 
4024  // Ignore inlined function arguments here.
4025  //
4026  // FIXME: Should we be checking DL->inlinedAt() to determine this?
4027  if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4028  return false;
4029 
4031  // Some arguments' frame index is recorded during argument lowering.
4032  if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
4033  Op = MachineOperand::CreateFI(FI);
4034 
4035  if (!Op && N.getNode()) {
4036  unsigned Reg;
4037  if (N.getOpcode() == ISD::CopyFromReg)
4038  Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
4039  else
4040  Reg = getTruncatedArgReg(N);
4041  if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4042  MachineRegisterInfo &RegInfo = MF.getRegInfo();
4043  unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4044  if (PR)
4045  Reg = PR;
4046  }
4047  if (Reg)
4048  Op = MachineOperand::CreateReg(Reg, false);
4049  }
4050 
4051  if (!Op) {
4052  // Check if ValueMap has reg number.
4054  if (VMI != FuncInfo.ValueMap.end())
4055  Op = MachineOperand::CreateReg(VMI->second, false);
4056  }
4057 
4058  if (!Op && N.getNode())
4059  // Check if frame index is available.
4060  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4061  if (FrameIndexSDNode *FINode =
4062  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4063  Op = MachineOperand::CreateFI(FINode->getIndex());
4064 
4065  if (!Op)
4066  return false;
4067 
4068  assert(Variable->isValidLocationForIntrinsic(DL) &&
4069  "Expected inlined-at fields to agree");
4070  if (Op->isReg())
4071  FuncInfo.ArgDbgValues.push_back(
4072  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4073  Op->getReg(), Offset, Variable, Expr));
4074  else
4075  FuncInfo.ArgDbgValues.push_back(
4076  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4077  .addOperand(*Op)
4078  .addImm(Offset)
4079  .addMetadata(Variable)
4080  .addMetadata(Expr));
4081 
4082  return true;
4083 }
4084 
4085 // VisualStudio defines setjmp as _setjmp
4086 #if defined(_MSC_VER) && defined(setjmp) && \
4087  !defined(setjmp_undefined_for_msvc)
4088 # pragma push_macro("setjmp")
4089 # undef setjmp
4090 # define setjmp_undefined_for_msvc
4091 #endif
4092 
4093 /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If
4094 /// we want to emit this as a call to a named external function, return the name
4095 /// otherwise lower it and return null.
4096 const char *
4097 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4098  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4099  SDLoc sdl = getCurSDLoc();
4100  DebugLoc dl = getCurDebugLoc();
4101  SDValue Res;
4102 
4103  switch (Intrinsic) {
4104  default:
4105  // By default, turn this into a target intrinsic node.
4106  visitTargetIntrinsic(I, Intrinsic);
4107  return nullptr;
4108  case Intrinsic::vastart: visitVAStart(I); return nullptr;
4109  case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4110  case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4111  case Intrinsic::returnaddress:
4114  getValue(I.getArgOperand(0))));
4115  return nullptr;
4116  case Intrinsic::frameaddress:
4119  getValue(I.getArgOperand(0))));
4120  return nullptr;
4121  case Intrinsic::read_register: {
4122  Value *Reg = I.getArgOperand(0);
4123  SDValue Chain = getRoot();
4124  SDValue RegName =
4125  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4126  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4127  Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4128  DAG.getVTList(VT, MVT::Other), Chain, RegName);
4129  setValue(&I, Res);
4130  DAG.setRoot(Res.getValue(1));
4131  return nullptr;
4132  }
4133  case Intrinsic::write_register: {
4134  Value *Reg = I.getArgOperand(0);
4135  Value *RegValue = I.getArgOperand(1);
4136  SDValue Chain = getRoot();
4137  SDValue RegName =
4138  DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4140  RegName, getValue(RegValue)));
4141  return nullptr;
4142  }
4143  case Intrinsic::setjmp:
4144  return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4145  case Intrinsic::longjmp:
4146  return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4147  case Intrinsic::memcpy: {
4148  // FIXME: this definition of "user defined address space" is x86-specific
4149  // Assert for address < 256 since we support only user defined address
4150  // spaces.
4151  assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
4152  < 256 &&
4153  cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
4154  < 256 &&
4155  "Unknown address space");
4156  SDValue Op1 = getValue(I.getArgOperand(0));
4157  SDValue Op2 = getValue(I.getArgOperand(1));
4158  SDValue Op3 = getValue(I.getArgOperand(2));
4159  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4160  if (!Align)
4161  Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4162  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4163  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4164  SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4165  false, isTC,
4168  updateDAGForMaybeTailCall(MC);
4169  return nullptr;
4170  }
4171  case Intrinsic::memset: {
4172  // FIXME: this definition of "user defined address space" is x86-specific
4173  // Assert for address < 256 since we support only user defined address
4174  // spaces.
4175  assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
4176  < 256 &&
4177  "Unknown address space");
4178  SDValue Op1 = getValue(I.getArgOperand(0));
4179  SDValue Op2 = getValue(I.getArgOperand(1));
4180  SDValue Op3 = getValue(I.getArgOperand(2));
4181  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4182  if (!Align)
4183  Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4184  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4185  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4186  SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4187  isTC, MachinePointerInfo(I.getArgOperand(0)));
4188  updateDAGForMaybeTailCall(MS);
4189  return nullptr;
4190  }
4191  case Intrinsic::memmove: {
4192  // FIXME: this definition of "user defined address space" is x86-specific
4193  // Assert for address < 256 since we support only user defined address
4194  // spaces.
4195  assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
4196  < 256 &&
4197  cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
4198  < 256 &&
4199  "Unknown address space");
4200  SDValue Op1 = getValue(I.getArgOperand(0));
4201  SDValue Op2 = getValue(I.getArgOperand(1));
4202  SDValue Op3 = getValue(I.getArgOperand(2));
4203  unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4204  if (!Align)
4205  Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4206  bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4207  bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4208  SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4209  isTC, MachinePointerInfo(I.getArgOperand(0)),
4211  updateDAGForMaybeTailCall(MM);
4212  return nullptr;
4213  }
4214  case Intrinsic::dbg_declare: {
4215  const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
4216  DILocalVariable *Variable = DI.getVariable();
4217  DIExpression *Expression = DI.getExpression();
4218  const Value *Address = DI.getAddress();
4219  assert(Variable && "Missing variable");
4220  if (!Address) {
4221  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4222  return nullptr;
4223  }
4224 
4225  // Check if address has undef value.
4226  if (isa<UndefValue>(Address) ||
4227  (Address->use_empty() && !isa<Argument>(Address))) {
4228  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4229  return nullptr;
4230  }
4231 
4232  SDValue &N = NodeMap[Address];
4233  if (!N.getNode() && isa<Argument>(Address))
4234  // Check unused arguments map.
4235  N = UnusedArgNodeMap[Address];
4236  SDDbgValue *SDV;
4237  if (N.getNode()) {
4238  if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
4239  Address = BCI->getOperand(0);
4240  // Parameters are handled specially.
4241  bool isParameter = Variable->getTag() == dwarf::DW_TAG_arg_variable ||
4242  isa<Argument>(Address);
4243 
4244  const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
4245 
4246  if (isParameter && !AI) {
4248  if (FINode)
4249  // Byval parameter. We have a frame index at this point.
4250  SDV = DAG.getFrameIndexDbgValue(
4251  Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
4252  else {
4253  // Address is an argument, so try to emit its dbg value using
4254  // virtual register info from the FuncInfo.ValueMap.
4255  EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
4256  N);
4257  return nullptr;
4258  }
4259  } else if (AI)
4260  SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
4261  true, 0, dl, SDNodeOrder);
4262  else {
4263  // Can't do anything with other non-AI cases yet.
4264  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4265  DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
4266  DEBUG(Address->dump());
4267  return nullptr;
4268  }
4269  DAG.AddDbgValue(SDV, N.getNode(), isParameter);
4270  } else {
4271  // If Address is an argument then try to emit its dbg value using
4272  // virtual register info from the FuncInfo.ValueMap.
4273  if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
4274  N)) {
4275  // If variable is pinned by a alloca in dominating bb then
4276  // use StaticAllocaMap.
4277  if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
4278  if (AI->getParent() != DI.getParent()) {
4280  FuncInfo.StaticAllocaMap.find(AI);
4281  if (SI != FuncInfo.StaticAllocaMap.end()) {
4282  SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
4283  0, dl, SDNodeOrder);
4284  DAG.AddDbgValue(SDV, nullptr, false);
4285  return nullptr;
4286  }
4287  }
4288  }
4289  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4290  }
4291  }
4292  return nullptr;
4293  }
4294  case Intrinsic::dbg_value: {
4295  const DbgValueInst &DI = cast<DbgValueInst>(I);
4296  assert(DI.getVariable() && "Missing variable");
4297 
4298  DILocalVariable *Variable = DI.getVariable();
4299  DIExpression *Expression = DI.getExpression();
4300  uint64_t Offset = DI.getOffset();
4301  const Value *V = DI.getValue();
4302  if (!V)
4303  return nullptr;
4304 
4305  SDDbgValue *SDV;
4306  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
4307  SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
4308  SDNodeOrder);
4309  DAG.AddDbgValue(SDV, nullptr, false);
4310  } else {
4311  // Do not use getValue() in here; we don't want to generate code at
4312  // this point if it hasn't been done yet.
4313  SDValue N = NodeMap[V];
4314  if (!N.getNode() && isa<Argument>(V))
4315  // Check unused arguments map.
4316  N = UnusedArgNodeMap[V];
4317  if (N.getNode()) {
4318  // A dbg.value for an alloca is always indirect.
4319  bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
4320  if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
4321  IsIndirect, N)) {
4322  SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
4323  IsIndirect, Offset, dl, SDNodeOrder);
4324  DAG.AddDbgValue(SDV, N.getNode(), false);
4325  }
4326  } else if (!V->use_empty() ) {
4327  // Do not call getValue(V) yet, as we don't want to generate code.
4328  // Remember it for later.
4329  DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
4330  DanglingDebugInfoMap[V] = DDI;
4331  } else {
4332  // We may expand this to cover more cases. One case where we have no
4333  // data available is an unreferenced parameter.
4334  DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4335  }
4336  }
4337 
4338  // Build a debug info table entry.
4339  if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
4340  V = BCI->getOperand(0);
4341  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
4342  // Don't handle byval struct arguments or VLAs, for example.
4343  if (!AI) {
4344  DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n");
4345  DEBUG(dbgs() << " Last seen at:\n " << *V << "\n");
4346  return nullptr;
4347  }
4349  FuncInfo.StaticAllocaMap.find(AI);
4350  if (SI == FuncInfo.StaticAllocaMap.end())
4351  return nullptr; // VLAs.
4352  return nullptr;
4353  }
4354 
4355  case Intrinsic::eh_typeid_for: {
4356  // Find the type id for the given typeinfo.
4358  unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
4359  Res = DAG.getConstant(TypeID, sdl, MVT::i32);
4360  setValue(&I, Res);
4361  return nullptr;
4362  }
4363 
4364  case Intrinsic::eh_return_i32:
4365  case Intrinsic::eh_return_i64:
4368  MVT::Other,
4369  getControlRoot(),
4370  getValue(I.getArgOperand(0)),
4371  getValue(I.getArgOperand(1))));
4372  return nullptr;
4373  case Intrinsic::eh_unwind_init:
4375  return nullptr;
4376  case Intrinsic::eh_dwarf_cfa: {
4377  SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
4378  TLI.getPointerTy(DAG.getDataLayout()));
4379  SDValue Offset = DAG.getNode(ISD::ADD, sdl,
4380  CfaArg.getValueType(),
4382  CfaArg.getValueType()),
4383  CfaArg);
4384  SDValue FA = DAG.getNode(
4386  DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
4387  setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
4388  FA, Offset));
4389  return nullptr;
4390  }
4391  case Intrinsic::eh_sjlj_callsite: {
4394  assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
4395  assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
4396 
4397  MMI.setCurrentCallSite(CI->getZExtValue());
4398  return nullptr;
4399  }
4400  case Intrinsic::eh_sjlj_functioncontext: {
4401  // Get and store the index of the function context.
4403  AllocaInst *FnCtx =
4404  cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
4405  int FI = FuncInfo.StaticAllocaMap[FnCtx];
4406  MFI->setFunctionContextIndex(FI);
4407  return nullptr;
4408  }
4409  case Intrinsic::eh_sjlj_setjmp: {
4410  SDValue Ops[2];
4411  Ops[0] = getRoot();
4412  Ops[1] = getValue(I.getArgOperand(0));
4414  DAG.getVTList(MVT::i32, MVT::Other), Ops);
4415  setValue(&I, Op.getValue(0));
4416  DAG.setRoot(Op.getValue(1));
4417  return nullptr;
4418  }
4419  case Intrinsic::eh_sjlj_longjmp: {
4421  getRoot(), getValue(I.getArgOperand(0))));
4422  return nullptr;
4423  }
4424 
4425  case Intrinsic::masked_gather:
4426  visitMaskedGather(I);
4427  return nullptr;
4428  case Intrinsic::masked_load:
4429  visitMaskedLoad(I);
4430  return nullptr;
4431  case Intrinsic::masked_scatter:
4432  visitMaskedScatter(I);
4433  return nullptr;
4434  case Intrinsic::masked_store:
4435  visitMaskedStore(I);
4436  return nullptr;
4437  case Intrinsic::x86_mmx_pslli_w:
4438  case Intrinsic::x86_mmx_pslli_d:
4439  case Intrinsic::x86_mmx_pslli_q:
4440  case Intrinsic::x86_mmx_psrli_w:
4441  case Intrinsic::x86_mmx_psrli_d:
4442  case Intrinsic::x86_mmx_psrli_q:
4443  case Intrinsic::x86_mmx_psrai_w:
4444  case Intrinsic::x86_mmx_psrai_d: {
4445  SDValue ShAmt = getValue(I.getArgOperand(1));
4446  if (isa<ConstantSDNode>(ShAmt)) {
4447  visitTargetIntrinsic(I, Intrinsic);
4448  return nullptr;
4449  }
4450  unsigned NewIntrinsic = 0;
4451  EVT ShAmtVT = MVT::v2i32;
4452  switch (Intrinsic) {
4453  case Intrinsic::x86_mmx_pslli_w:
4454  NewIntrinsic = Intrinsic::x86_mmx_psll_w;
4455  break;
4456  case Intrinsic::x86_mmx_pslli_d:
4457  NewIntrinsic = Intrinsic::x86_mmx_psll_d;
4458  break;
4459  case Intrinsic::x86_mmx_pslli_q:
4460  NewIntrinsic = Intrinsic::x86_mmx_psll_q;
4461  break;
4462  case Intrinsic::x86_mmx_psrli_w:
4463  NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
4464  break;
4465  case Intrinsic::x86_mmx_psrli_d:
4466  NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
4467  break;
4468  case Intrinsic::x86_mmx_psrli_q:
4469  NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
4470  break;
4471  case Intrinsic::x86_mmx_psrai_w:
4472  NewIntrinsic = Intrinsic::x86_mmx_psra_w;
4473  break;
4474  case Intrinsic::x86_mmx_psrai_d:
4475  NewIntrinsic = Intrinsic::x86_mmx_psra_d;
4476  break;
4477  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
4478  }
4479 
4480  // The vector shift intrinsics with scalars uses 32b shift amounts but
4481  // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
4482  // to be zero.
4483  // We must do this early because v2i32 is not a legal type.
4484  SDValue ShOps[2];
4485  ShOps[0] = ShAmt;
4486  ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
4487  ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
4488  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4489  ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
4490  Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
4491  DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
4492  getValue(I.getArgOperand(0)), ShAmt);
4493  setValue(&I, Res);
4494  return nullptr;
4495  }
4496  case Intrinsic::convertff:
4497  case Intrinsic::convertfsi:
4498  case Intrinsic::convertfui:
4499  case Intrinsic::convertsif:
4500  case Intrinsic::convertuif:
4501  case Intrinsic::convertss:
4502  case Intrinsic::convertsu:
4503  case Intrinsic::convertus:
4504  case Intrinsic::convertuu: {
4506  switch (Intrinsic) {
4507  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
4508  case Intrinsic::convertff: Code = ISD::CVT_FF; break;
4509  case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
4510  case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
4511  case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
4512  case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
4513  case Intrinsic::convertss: Code = ISD::CVT_SS; break;
4514  case Intrinsic::convertsu: Code = ISD::CVT_SU; break;
4515  case Intrinsic::convertus: Code = ISD::CVT_US; break;
4516  case Intrinsic::convertuu: Code = ISD::CVT_UU; break;
4517  }
4518  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4519  const Value *Op1 = I.getArgOperand(0);
4520  Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
4521  DAG.getValueType(DestVT),
4522  DAG.getValueType(getValue(Op1).getValueType()),
4523  getValue(I.getArgOperand(1)),
4524  getValue(I.getArgOperand(2)),
4525  Code);
4526  setValue(&I, Res);
4527  return nullptr;
4528  }
4529  case Intrinsic::powi:
4530  setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
4531  getValue(I.getArgOperand(1)), DAG));
4532  return nullptr;
4533  case Intrinsic::log:
4534  setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
4535  return nullptr;
4536  case Intrinsic::log2:
4537  setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
4538  return nullptr;
4539  case Intrinsic::log10:
4540  setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
4541  return nullptr;
4542  case Intrinsic::exp:
4543  setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
4544  return nullptr;
4545  case Intrinsic::exp2:
4546  setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
4547  return nullptr;
4548  case Intrinsic::pow:
4549  setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
4550  getValue(I.getArgOperand(1)), DAG, TLI));
4551  return nullptr;
4552  case Intrinsic::sqrt:
4553  case Intrinsic::fabs:
4554  case Intrinsic::sin:
4555  case Intrinsic::cos:
4556  case Intrinsic::floor:
4557  case Intrinsic::ceil:
4558  case Intrinsic::trunc:
4559  case Intrinsic::rint:
4560  case Intrinsic::nearbyint:
4561  case Intrinsic::round: {
4562  unsigned Opcode;
4563  switch (Intrinsic) {
4564  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
4565  case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
4566  case Intrinsic::fabs: Opcode = ISD::FABS; break;
4567  case Intrinsic::sin: Opcode = ISD::FSIN; break;
4568  case Intrinsic::cos: Opcode = ISD::FCOS; break;
4569  case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
4570  case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
4571  case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
4572  case Intrinsic::rint: Opcode = ISD::FRINT; break;
4573  case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
4574  case Intrinsic::round: Opcode = ISD::FROUND; break;
4575  }
4576 
4577  setValue(&I, DAG.getNode(Opcode, sdl,
4578  getValue(I.getArgOperand(0)).getValueType(),
4579  getValue(I.getArgOperand(0))));
4580  return nullptr;
4581  }
4582  case Intrinsic::minnum:
4583  setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
4584  getValue(I.getArgOperand(0)).getValueType(),
4585  getValue(I.getArgOperand(0)),
4586  getValue(I.getArgOperand(1))));
4587  return nullptr;
4588  case Intrinsic::maxnum:
4589  setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
4590  getValue(I.getArgOperand(0)).getValueType(),
4591  getValue(I.getArgOperand(0)),
4592  getValue(I.getArgOperand(1))));
4593  return nullptr;
4594  case Intrinsic::copysign:
4596  getValue(I.getArgOperand(0)).getValueType(),
4597  getValue(I.getArgOperand(0)),
4598  getValue(I.getArgOperand(1))));
4599  return nullptr;
4600  case Intrinsic::fma:
4601  setValue(&I, DAG.getNode(ISD::FMA, sdl,
4602  getValue(I.getArgOperand(0)).getValueType(),
4603  getValue(I.getArgOperand(0)),
4604  getValue(I.getArgOperand(1)),
4605  getValue(I.getArgOperand(2))));
4606  return nullptr;
4607  case Intrinsic::fmuladd: {
4608  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4609  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
4610  TLI.isFMAFasterThanFMulAndFAdd(VT)) {
4611  setValue(&I, DAG.getNode(ISD::FMA, sdl,
4612  getValue(I.getArgOperand(0)).getValueType(),
4613  getValue(I.getArgOperand(0)),
4614  getValue(I.getArgOperand(1)),
4615  getValue(I.getArgOperand(2))));
4616  } else {
4617  SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
4618  getValue(I.getArgOperand(0)).getValueType(),
4619  getValue(I.getArgOperand(0)),
4620  getValue(I.getArgOperand(1)));
4621  SDValue Add = DAG.getNode(ISD::FADD, sdl,
4622  getValue(I.getArgOperand(0)).getValueType(),
4623  Mul,
4624  getValue(I.getArgOperand(2)));
4625  setValue(&I, Add);
4626  }
4627  return nullptr;
4628  }
4629  case Intrinsic::convert_to_fp16:
4632  getValue(I.getArgOperand(0)),
4633  DAG.getTargetConstant(0, sdl,
4634  MVT::i32))));
4635  return nullptr;
4636  case Intrinsic::convert_from_fp16:
4638  TLI.getValueType(DAG.getDataLayout(), I.getType()),
4640  getValue(I.getArgOperand(0)))));
4641  return nullptr;
4642  case Intrinsic::pcmarker: {
4643  SDValue Tmp = getValue(I.getArgOperand(0));
4645  return nullptr;
4646  }
4647  case Intrinsic::readcyclecounter: {
4648  SDValue Op = getRoot();
4649  Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
4651  setValue(&I, Res);
4652  DAG.setRoot(Res.getValue(1));
4653  return nullptr;
4654  }
4655  case Intrinsic::bswap:
4656  setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
4657  getValue(I.getArgOperand(0)).getValueType(),
4658  getValue(I.getArgOperand(0))));
4659  return nullptr;
4660  case Intrinsic::cttz: {
4661  SDValue Arg = getValue(I.getArgOperand(0));
4662  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
4663  EVT Ty = Arg.getValueType();
4664  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
4665  sdl, Ty, Arg));
4666  return nullptr;
4667  }
4668  case Intrinsic::ctlz: {
4669  SDValue Arg = getValue(I.getArgOperand(0));
4670  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
4671  EVT Ty = Arg.getValueType();
4672  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
4673  sdl, Ty, Arg));
4674  return nullptr;
4675  }
4676  case Intrinsic::ctpop: {
4677  SDValue Arg = getValue(I.getArgOperand(0));
4678  EVT Ty = Arg.getValueType();
4679  setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
4680  return nullptr;
4681  }
4682  case Intrinsic::stacksave: {
4683  SDValue Op = getRoot();
4684  Res = DAG.getNode(
4685  ISD::STACKSAVE, sdl,
4687  setValue(&I, Res);
4688  DAG.setRoot(Res.getValue(1));
4689  return nullptr;
4690  }
4691  case Intrinsic::stackrestore: {
4692  Res = getValue(I.getArgOperand(0));
4694  return nullptr;
4695  }
4696  case Intrinsic::stackprotector: {
4697  // Emit code into the DAG to store the stack guard onto the stack.
4699  MachineFrameInfo *MFI = MF.getFrameInfo();
4700  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
4701  SDValue Src, Chain = getRoot();
4702  const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
4703  const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
4704 
4705  // See if Ptr is a bitcast. If it is, look through it and see if we can get
4706  // global variable __stack_chk_guard.
4707  if (!GV)
4708  if (const Operator *BC = dyn_cast<Operator>(Ptr))
4709  if (BC->getOpcode() == Instruction::BitCast)
4710  GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
4711 
4712  if (GV && TLI.useLoadStackGuardNode()) {
4713  // Emit a LOAD_STACK_GUARD node.
4715  sdl, PtrTy, Chain);
4716  MachinePointerInfo MPInfo(GV);
4718  unsigned Flags = MachineMemOperand::MOLoad |
4720  *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
4721  PtrTy.getSizeInBits() / 8,
4722  DAG.getEVTAlignment(PtrTy));
4723  Node->setMemRefs(MemRefs, MemRefs + 1);
4724 
4725  // Copy the guard value to a virtual register so that it can be
4726  // retrieved in the epilogue.
4727  Src = SDValue(Node, 0);
4728  const TargetRegisterClass *RC =
4730  unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
4731 
4732  SPDescriptor.setGuardReg(Reg);
4733  Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
4734  } else {
4735  Src = getValue(I.getArgOperand(0)); // The guard's value.
4736  }
4737 
4738  AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
4739 
4740  int FI = FuncInfo.StaticAllocaMap[Slot];
4741  MFI->setStackProtectorIndex(FI);
4742 
4743  SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
4744 
4745  // Store the stack protector onto the stack.
4746  Res = DAG.getStore(Chain, sdl, Src, FIN,
4748  true, false, 0);
4749  setValue(&I, Res);
4750  DAG.setRoot(Res);
4751  return nullptr;
4752  }
4753  case Intrinsic::objectsize: {
4754  // If we don't know by now, we're never going to know.
4756 
4757  assert(CI && "Non-constant type in __builtin_object_size?");
4758 
4759  SDValue Arg = getValue(I.getCalledValue());
4760  EVT Ty = Arg.getValueType();
4761 
4762  if (CI->isZero())
4763  Res = DAG.getConstant(-1ULL, sdl, Ty);
4764  else
4765  Res = DAG.getConstant(0, sdl, Ty);
4766 
4767  setValue(&I, Res);
4768  return nullptr;
4769  }
4770  case Intrinsic::annotation:
4771  case Intrinsic::ptr_annotation:
4772  // Drop the intrinsic, but forward the value
4773  setValue(&I, getValue(I.getOperand(0)));
4774  return nullptr;
4775  case Intrinsic::assume:
4776  case Intrinsic::var_annotation:
4777  // Discard annotate attributes and assumptions
4778  return nullptr;
4779 
4780  case Intrinsic::init_trampoline: {
4781  const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
4782 
4783  SDValue Ops[6];
4784  Ops[0] = getRoot();
4785  Ops[1] = getValue(I.getArgOperand(0));
4786  Ops[2] = getValue(I.getArgOperand(1));
4787  Ops[3] = getValue(I.getArgOperand(2));
4788  Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
4789  Ops[5] = DAG.getSrcValue(F);
4790 
4791  Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
4792 
4793  DAG.setRoot(Res);
4794  return nullptr;
4795  }
4796  case Intrinsic::adjust_trampoline: {
4799  getValue(I.getArgOperand(0))));
4800  return nullptr;
4801  }
4802  case Intrinsic::gcroot:
4803  if (GFI) {
4804  const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
4805  const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
4806 
4807  FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
4808  GFI->addStackRoot(FI->getIndex(), TypeMap);
4809  }
4810  return nullptr;
4811  case Intrinsic::gcread:
4812  case Intrinsic::gcwrite:
4813  llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
4814  case Intrinsic::flt_rounds:
4816  return nullptr;
4817 
4818  case Intrinsic::expect: {
4819  // Just replace __builtin_expect(exp, c) with EXP.
4820  setValue(&I, getValue(I.getArgOperand(0)));
4821  return nullptr;
4822  }
4823 
4824  case Intrinsic::debugtrap:
4825  case Intrinsic::trap: {
4827  I.getAttributes()
4828  .getAttribute(AttributeSet::FunctionIndex, "trap-func-name")
4829  .getValueAsString();
4830  if (TrapFuncName.empty()) {
4831  ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
4833  DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
4834  return nullptr;
4835  }
4837 
4839  CLI.setDebugLoc(sdl).setChain(getRoot()).setCallee(
4840  CallingConv::C, I.getType(),
4841  DAG.getExternalSymbol(TrapFuncName.data(),
4842  TLI.getPointerTy(DAG.getDataLayout())),
4843  std::move(Args), 0);
4844 
4845  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
4846  DAG.setRoot(Result.second);
4847  return nullptr;
4848  }
4849 
4850  case Intrinsic::uadd_with_overflow:
4851  case Intrinsic::sadd_with_overflow:
4852  case Intrinsic::usub_with_overflow:
4853  case Intrinsic::ssub_with_overflow:
4854  case Intrinsic::umul_with_overflow:
4855  case Intrinsic::smul_with_overflow: {
4856  ISD::NodeType Op;
4857  switch (Intrinsic) {
4858  default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
4859  case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
4860  case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
4861  case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
4862  case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
4863  case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
4864  case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
4865  }
4866  SDValue Op1 = getValue(I.getArgOperand(0));
4867  SDValue Op2 = getValue(I.getArgOperand(1));
4868 
4869  SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
4870  setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
4871  return nullptr;
4872  }
4873  case Intrinsic::prefetch: {
4874  SDValue Ops[5];
4875  unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4876  Ops[0] = getRoot();
4877  Ops[1] = getValue(I.getArgOperand(0));
4878  Ops[2] = getValue(I.getArgOperand(1));
4879  Ops[3] = getValue(I.getArgOperand(2));
4880  Ops[4] = getValue(I.getArgOperand(3));
4882  DAG.getVTList(MVT::Other), Ops,
4883  EVT::getIntegerVT(*Context, 8),
4885  0, /* align */
4886  false, /* volatile */
4887  rw==0, /* read */
4888  rw==1)); /* write */
4889  return nullptr;
4890  }
4891  case Intrinsic::lifetime_start:
4892  case Intrinsic::lifetime_end: {
4893  bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
4894  // Stack coloring is not enabled in O0, discard region information.
4895  if (TM.getOptLevel() == CodeGenOpt::None)
4896  return nullptr;
4897 
4898  SmallVector<Value *, 4> Allocas;
4899  GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
4900 
4901  for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
4902  E = Allocas.end(); Object != E; ++Object) {
4903  AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
4904 
4905  // Could not find an Alloca.
4906  if (!LifetimeObject)
4907  continue;
4908 
4909  // First check that the Alloca is static, otherwise it won't have a
4910  // valid frame index.
4911  auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
4912  if (SI == FuncInfo.StaticAllocaMap.end())
4913  return nullptr;
4914 
4915  int FI = SI->second;
4916 
4917  SDValue Ops[2];
4918  Ops[0] = getRoot();
4919  Ops[1] =
4920  DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout()), true);
4921  unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
4922 
4923  Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
4924  DAG.setRoot(Res);
4925  }
4926  return nullptr;
4927  }
4928  case Intrinsic::invariant_start:
4929  // Discard region information.
4931  return nullptr;
4932  case Intrinsic::invariant_end:
4933  // Discard region information.
4934  return nullptr;
4935  case Intrinsic::stackprotectorcheck: {
4936  // Do not actually emit anything for this basic block. Instead we initialize
4937  // the stack protector descriptor and export the guard variable so we can
4938  // access it in FinishBasicBlock.
4939  const BasicBlock *BB = I.getParent();
4940  SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
4942 
4943  // Flush our exports since we are going to process a terminator.
4944  (void)getControlRoot();
4945  return nullptr;
4946  }
4947  case Intrinsic::clear_cache:
4948  return TLI.getClearCacheBuiltinName();
4949  case Intrinsic::eh_actions:
4951  return nullptr;
4952  case Intrinsic::donothing:
4953  // ignore
4954  return nullptr;
4955  case Intrinsic::experimental_stackmap: {
4956  visitStackmap(I);
4957  return nullptr;
4958  }
4959  case Intrinsic::experimental_patchpoint_void:
4960  case Intrinsic::experimental_patchpoint_i64: {
4961  visitPatchpoint(&I);
4962  return nullptr;
4963  }
4964  case Intrinsic::experimental_gc_statepoint: {
4965  visitStatepoint(I);
4966  return nullptr;
4967  }
4968  case Intrinsic::experimental_gc_result_int:
4969  case Intrinsic::experimental_gc_result_float:
4970  case Intrinsic::experimental_gc_result_ptr:
4971  case Intrinsic::experimental_gc_result: {
4972  visitGCResult(I);
4973  return nullptr;
4974  }
4975  case Intrinsic::experimental_gc_relocate: {
4976  visitGCRelocate(I);
4977  return nullptr;
4978  }
4979  case Intrinsic::instrprof_increment:
4980  llvm_unreachable("instrprof failed to lower an increment");
4981 
4982  case Intrinsic::localescape: {
4984  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4985 
4986  // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
4987  // is the same on all targets.
4988  for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
4989  Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
4990  if (isa<ConstantPointerNull>(Arg))
4991  continue; // Skip null pointers. They represent a hole in index space.
4992  AllocaInst *Slot = cast<AllocaInst>(Arg);
4993  assert(FuncInfo.StaticAllocaMap.count(Slot) &&
4994  "can only escape static allocas");
4995  int FI = FuncInfo.StaticAllocaMap[Slot];
4996  MCSymbol *FrameAllocSym =
5001  .addSym(FrameAllocSym)
5002  .addFrameIndex(FI);
5003  }
5004 
5005  return nullptr;
5006  }
5007 
5008  case Intrinsic::localrecover: {
5009  // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5011  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5012 
5013  // Get the symbol that defines the frame offset.
5014  auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5015  auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5016  unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
5017  MCSymbol *FrameAllocSym =
5019  GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
5020 
5021  // Create a MCSymbol for the label to avoid any target lowering
5022  // that would make this PC relative.
5023  SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5024  SDValue OffsetVal =
5025  DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5026 
5027  // Add the offset to the FP.
5028  Value *FP = I.getArgOperand(1);
5029  SDValue FPVal = getValue(FP);
5030  SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5031  setValue(&I, Add);
5032 
5033  return nullptr;
5034  }
5035  case Intrinsic::eh_begincatch:
5036  case Intrinsic::eh_endcatch:
5037  llvm_unreachable("begin/end catch intrinsics not lowered in codegen");
5038  case Intrinsic::eh_exceptioncode: {
5039  unsigned Reg = TLI.getExceptionPointerRegister();
5040  assert(Reg && "cannot get exception code on this platform");
5041  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5042  const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5043  assert(FuncInfo.MBB->isLandingPad() && "eh.exceptioncode in non-lpad");
5044  unsigned VReg = FuncInfo.MBB->addLiveIn(Reg, PtrRC);
5045  SDValue N =
5046  DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5048  setValue(&I, N);
5049  return nullptr;
5050  }
5051  }
5052 }
5053 
5054 std::pair<SDValue, SDValue>
5055 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
5056  MachineBasicBlock *LandingPad) {
5058  MCSymbol *BeginLabel = nullptr;
5059 
5060  if (LandingPad) {
5061  // Insert a label before the invoke call to mark the try range. This can be
5062  // used to detect deletion of the invoke via the MachineModuleInfo.
5063  BeginLabel = MMI.getContext().createTempSymbol();
5064 
5065  // For SjLj, keep track of which landing pads go with which invokes
5066  // so as to maintain the ordering of pads in the LSDA.
5067  unsigned CallSiteIndex = MMI.getCurrentCallSite();
5068  if (CallSiteIndex) {
5069  MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
5070  LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
5071 
5072  // Now that the call site is handled, stop tracking it.
5073  MMI.setCurrentCallSite(0);
5074  }
5075 
5076  // Both PendingLoads and PendingExports must be flushed here;
5077  // this call might not return.
5078  (void)getRoot();
5079  DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
5080 
5081  CLI.setChain(getRoot());
5082  }
5083  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5084  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5085 
5086  assert((CLI.IsTailCall || Result.second.getNode()) &&
5087  "Non-null chain expected with non-tail call!");
5088  assert((Result.second.getNode() || !Result.first.getNode()) &&
5089  "Null value expected with tail call!");
5090 
5091  if (!Result.second.getNode()) {
5092  // As a special case, a null chain means that a tail call has been emitted
5093  // and the DAG root is already updated.
5094  HasTailCall = true;
5095 
5096  // Since there's no actual continuation from this block, nothing can be
5097  // relying on us setting vregs for them.
5098  PendingExports.clear();
5099  } else {
5100  DAG.setRoot(Result.second);
5101  }
5102 
5103  if (LandingPad) {
5104  // Insert a label at the end of the invoke call to mark the try range. This
5105  // can be used to detect deletion of the invoke via the MachineModuleInfo.
5106  MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
5107  DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
5108 
5109  // Inform MachineModuleInfo of range.
5110  MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
5111  }
5112 
5113  return Result;
5114 }
5115 
5117  bool isTailCall,
5118  MachineBasicBlock *LandingPad) {
5119  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
5120  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
5121  Type *RetTy = FTy->getReturnType();
5122 
5125  Args.reserve(CS.arg_size());
5126 
5127  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
5128  i != e; ++i) {
5129  const Value *V = *i;
5130 
5131  // Skip empty types
5132  if (V->getType()->isEmptyTy())
5133  continue;
5134 
5135  SDValue ArgNode = getValue(V);
5136  Entry.Node = ArgNode; Entry.Ty = V->getType();
5137 
5138  // Skip the first return-type Attribute to get to params.
5139  Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
5140  Args.push_back(Entry);
5141 
5142  // If we have an explicit sret argument that is an Instruction, (i.e., it
5143  // might point to function-local memory), we can't meaningfully tail-call.
5144  if (Entry.isSRet && isa<Instruction>(V))
5145  isTailCall = false;
5146  }
5147 
5148  // Check if target-independent constraints permit a tail call here.
5149  // Target-dependent constraints are checked within TLI->LowerCallTo.
5150  if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
5151  isTailCall = false;
5152 
5155  .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
5156  .setTailCall(isTailCall);
5157  std::pair<SDValue,SDValue> Result = lowerInvokable(CLI, LandingPad);
5158 
5159  if (Result.first.getNode())
5160  setValue(CS.getInstruction(), Result.first);
5161 }
5162 
5163 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
5164 /// value is equal or not-equal to zero.
5166  for (const User *U : V->users()) {
5167  if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
5168  if (IC->isEquality())
5169  if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
5170  if (C->isNullValue())
5171  continue;
5172  // Unknown instruction.
5173  return false;
5174  }
5175  return true;
5176 }
5177 
5178 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
5179  Type *LoadTy,
5180  SelectionDAGBuilder &Builder) {
5181 
5182  // Check to see if this load can be trivially constant folded, e.g. if the
5183  // input is from a string literal.
5184  if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
5185  // Cast pointer to the type we really want to load.
5186  LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
5187  PointerType::getUnqual(LoadTy));
5188 
5189  if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
5190  const_cast<Constant *>(LoadInput), *Builder.DL))
5191  return Builder.getValue(LoadCst);
5192  }
5193 
5194  // Otherwise, we have to emit the load. If the pointer is to unfoldable but
5195  // still constant memory, the input chain can be the entry node.
5196  SDValue Root;
5197  bool ConstantMemory = false;
5198 
5199  // Do not serialize (non-volatile) loads of constant memory with anything.
5200  if (Builder.AA->pointsToConstantMemory(PtrVal)) {
5201  Root = Builder.DAG.getEntryNode();
5202  ConstantMemory = true;
5203  } else {
5204  // Do not serialize non-volatile loads against each other.
5205  Root = Builder.DAG.getRoot();
5206  }
5207 
5208  SDValue Ptr = Builder.getValue(PtrVal);
5209  SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
5210  Ptr, MachinePointerInfo(PtrVal),
5211  false /*volatile*/,
5212  false /*nontemporal*/,
5213  false /*isinvariant*/, 1 /* align=1 */);
5214 
5215  if (!ConstantMemory)
5216  Builder.PendingLoads.push_back(LoadVal.getValue(1));
5217  return LoadVal;
5218 }
5219 
5220 /// processIntegerCallValue - Record the value for an instruction that
5221 /// produces an integer result, converting the type where necessary.
5222 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
5223  SDValue Value,
5224  bool IsSigned) {
5226  I.getType(), true);
5227  if (IsSigned)
5228  Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
5229  else
5230  Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
5231  setValue(&I, Value);
5232 }
5233 
5234 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
5235 /// If so, return true and lower it, otherwise return false and it will be
5236 /// lowered like a normal call.
5237 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
5238  // Verify that the prototype makes sense. int memcmp(void*,void*,size_t)
5239  if (I.getNumArgOperands() != 3)
5240  return false;
5241 
5242  const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
5243  if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
5244  !I.getArgOperand(2)->getType()->isIntegerTy() ||
5245  !I.getType()->isIntegerTy())
5246  return false;
5247 
5248  const Value *Size = I.getArgOperand(2);
5249  const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
5250  if (CSize && CSize->getZExtValue() == 0) {
5252  I.getType(), true);
5253  setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
5254  return true;
5255  }
5256 
5258  std::pair<SDValue, SDValue> Res =
5260  getValue(LHS), getValue(RHS), getValue(Size),
5261  MachinePointerInfo(LHS),
5262  MachinePointerInfo(RHS));
5263  if (Res.first.getNode()) {
5264  processIntegerCallValue(I, Res.first, true);
5265  PendingLoads.push_back(Res.second);
5266  return true;
5267  }
5268 
5269  // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
5270  // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
5271  if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
5272  bool ActuallyDoIt = true;
5273  MVT LoadVT;
5274  Type *LoadTy;
5275  switch (CSize->getZExtValue()) {
5276  default:
5277  LoadVT = MVT::Other;
5278  LoadTy = nullptr;
5279  ActuallyDoIt = false;
5280  break;
5281  case 2:
5282  LoadVT = MVT::i16;
5283  LoadTy = Type::getInt16Ty(CSize->getContext());
5284  break;
5285  case 4:
5286  LoadVT = MVT::i32;
5287  LoadTy = Type::getInt32Ty(CSize->getContext());
5288  break;
5289  case 8:
5290  LoadVT = MVT::i64;
5291  LoadTy = Type::getInt64Ty(CSize->getContext());
5292  break;
5293  /*
5294  case 16:
5295  LoadVT = MVT::v4i32;
5296  LoadTy = Type::getInt32Ty(CSize->getContext());
5297  LoadTy = VectorType::get(LoadTy, 4);
5298  break;
5299  */
5300  }
5301 
5302  // This turns into unaligned loads. We only do this if the target natively
5303  // supports the MVT we'll be loading or if it is small enough (<= 4) that
5304  // we'll only produce a small number of byte loads.
5305 
5306  // Require that we can find a legal MVT, and only do this if the target
5307  // supports unaligned loads of that type. Expanding into byte loads would
5308  // bloat the code.
5309  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5310  if (ActuallyDoIt && CSize->getZExtValue() > 4) {
5311  unsigned DstAS = LHS->getType()->getPointerAddressSpace();
5312  unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
5313  // TODO: Handle 5 byte compare as 4-byte + 1 byte.
5314  // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
5315  // TODO: Check alignment of src and dest ptrs.
5316  if (!TLI.isTypeLegal(LoadVT) ||
5317  !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
5318  !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
5319  ActuallyDoIt = false;
5320  }
5321 
5322  if (ActuallyDoIt) {
5323  SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
5324  SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
5325 
5326  SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
5327  ISD::SETNE);
5328  processIntegerCallValue(I, Res, false);
5329  return true;
5330  }
5331  }
5332 
5333 
5334  return false;
5335 }
5336 
5337 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
5338 /// form. If so, return true and lower it, otherwise return false and it
5339 /// will be lowered like a normal call.
5340 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
5341  // Verify that the prototype makes sense. void *memchr(void *, int, size_t)
5342  if (I.getNumArgOperands() != 3)
5343  return false;
5344 
5345  const Value *Src = I.getArgOperand(0);
5346  const Value *Char = I.getArgOperand(1);
5347  const Value *Length = I.getArgOperand(2);
5348  if (!Src->getType()->isPointerTy() ||
5349  !Char->getType()->isIntegerTy() ||
5350  !Length->getType()->isIntegerTy() ||
5351  !I.getType()->isPointerTy())
5352  return false;
5353 
5355  std::pair<SDValue, SDValue> Res =
5357  getValue(Src), getValue(Char), getValue(Length),
5358  MachinePointerInfo(Src));
5359  if (Res.first.getNode()) {
5360  setValue(&I, Res.first);
5361  PendingLoads.push_back(Res.second);
5362  return true;
5363  }
5364 
5365  return false;
5366 }
5367 
5368 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
5369 /// optimized form. If so, return true and lower it, otherwise return false
5370 /// and it will be lowered like a normal call.
5371 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
5372  // Verify that the prototype makes sense. char *strcpy(char *, char *)
5373  if (I.getNumArgOperands() != 2)
5374  return false;
5375 
5376  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
5377  if (!Arg0->getType()->isPointerTy() ||
5378  !Arg1->getType()->isPointerTy() ||
5379  !I.getType()->isPointerTy())
5380  return false;
5381 
5383  std::pair<SDValue, SDValue> Res =
5385  getValue(Arg0), getValue(Arg1),
5386  MachinePointerInfo(Arg0),
5387  MachinePointerInfo(Arg1), isStpcpy);
5388  if (Res.first.getNode()) {
5389  setValue(&I, Res.first);
5390  DAG.setRoot(Res.second);
5391  return true;
5392  }
5393 
5394  return false;
5395 }
5396 
5397 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
5398 /// If so, return true and lower it, otherwise return false and it will be
5399 /// lowered like a normal call.
5400 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
5401  // Verify that the prototype makes sense. int strcmp(void*,void*)
5402  if (I.getNumArgOperands() != 2)
5403  return false;
5404 
5405  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
5406  if (!Arg0->getType()->isPointerTy() ||
5407  !Arg1->getType()->isPointerTy() ||
5408  !I.getType()->isIntegerTy())
5409  return false;
5410 
5412  std::pair<SDValue, SDValue> Res =
5414  getValue(Arg0), getValue(Arg1),
5415  MachinePointerInfo(Arg0),
5416  MachinePointerInfo(Arg1));
5417  if (Res.first.getNode()) {
5418  processIntegerCallValue(I, Res.first, true);
5419  PendingLoads.push_back(Res.second);
5420  return true;
5421  }
5422 
5423  return false;
5424 }
5425 
5426 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
5427 /// form. If so, return true and lower it, otherwise return false and it
5428 /// will be lowered like a normal call.
5429 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
5430  // Verify that the prototype makes sense. size_t strlen(char *)
5431  if (I.getNumArgOperands() != 1)
5432  return false;
5433 
5434  const Value *Arg0 = I.getArgOperand(0);
5435  if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
5436  return false;
5437 
5439  std::pair<SDValue, SDValue> Res =
5441  getValue(Arg0), MachinePointerInfo(Arg0));
5442  if (Res.first.getNode()) {
5443  processIntegerCallValue(I, Res.first, false);
5444  PendingLoads.push_back(Res.second);
5445  return true;
5446  }
5447 
5448  return false;
5449 }
5450 
5451 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
5452 /// form. If so, return true and lower it, otherwise return false and it
5453 /// will be lowered like a normal call.
5454 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
5455  // Verify that the prototype makes sense. size_t strnlen(char *, size_t)
5456  if (I.getNumArgOperands() != 2)
5457  return false;
5458 
5459  const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
5460  if (!Arg0->getType()->isPointerTy() ||
5461  !Arg1->getType()->isIntegerTy() ||
5462  !I.getType()->isIntegerTy())
5463  return false;
5464 
5466  std::pair<SDValue, SDValue> Res =
5468  getValue(Arg0), getValue(Arg1),
5469  MachinePointerInfo(Arg0));
5470  if (Res.first.getNode()) {
5471  processIntegerCallValue(I, Res.first, false);
5472  PendingLoads.push_back(Res.second);
5473  return true;
5474  }
5475 
5476  return false;
5477 }
5478 
5479 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
5480 /// operation (as expected), translate it to an SDNode with the specified opcode
5481 /// and return true.
5482 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
5483  unsigned Opcode) {
5484  // Sanity check that it really is a unary floating-point call.
5485  if (I.getNumArgOperands() != 1 ||
5486  !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
5487  I.getType() != I.getArgOperand(0)->getType() ||
5488  !I.onlyReadsMemory())
5489  return false;
5490 
5491  SDValue Tmp = getValue(I.getArgOperand(0));
5492  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
5493  return true;
5494 }
5495 
5496 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
5497 /// operation (as expected), translate it to an SDNode with the specified opcode
5498 /// and return true.
5499 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
5500  unsigned Opcode) {
5501  // Sanity check that it really is a binary floating-point call.
5502  if (I.getNumArgOperands() != 2 ||
5503  !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
5504  I.getType() != I.getArgOperand(0)->getType() ||
5505  I.getType() != I.getArgOperand(1)->getType() ||
5506  !I.onlyReadsMemory())
5507  return false;
5508 
5509  SDValue Tmp0 = getValue(I.getArgOperand(0));
5510  SDValue Tmp1 = getValue(I.getArgOperand(1));
5511  EVT VT = Tmp0.getValueType();
5512  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
5513  return true;
5514 }
5515 
5516 void SelectionDAGBuilder::visitCall(const CallInst &I) {
5517  // Handle inline assembly differently.
5518  if (isa<InlineAsm>(I.getCalledValue())) {
5519  visitInlineAsm(&I);
5520  return;
5521  }
5522 
5524  ComputeUsesVAFloatArgument(I, &MMI);
5525 
5526  const char *RenameFn = nullptr;
5527  if (Function *F = I.getCalledFunction()) {
5528  if (F->isDeclaration()) {
5529  if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
5530  if (unsigned IID = II->getIntrinsicID(F)) {
5531  RenameFn = visitIntrinsicCall(I, IID);
5532  if (!RenameFn)
5533  return;
5534  }
5535  }
5536  if (Intrinsic::ID IID = F->getIntrinsicID()) {
5537  RenameFn = visitIntrinsicCall(I, IID);
5538  if (!RenameFn)
5539  return;
5540  }
5541  }
5542 
5543  // Check for well-known libc/libm calls. If the function is internal, it
5544  // can't be a library call.
5546  if (!F->hasLocalLinkage() && F->hasName() &&
5547  LibInfo->getLibFunc(F->getName(), Func) &&
5548  LibInfo->hasOptimizedCodeGen(Func)) {
5549  switch (Func) {
5550  default: break;
5551  case LibFunc::copysign:
5552  case LibFunc::copysignf:
5553  case LibFunc::copysignl:
5554  if (I.getNumArgOperands() == 2 && // Basic sanity checks.
5556  I.getType() == I.getArgOperand(0)->getType() &&
5557  I.getType() == I.getArgOperand(1)->getType() &&
5558  I.onlyReadsMemory()) {
5559  SDValue LHS = getValue(I.getArgOperand(0));
5560  SDValue RHS = getValue(I.getArgOperand(1));
5562  LHS.getValueType(), LHS, RHS));
5563  return;
5564  }
5565  break;
5566  case LibFunc::fabs:
5567  case LibFunc::fabsf:
5568  case LibFunc::fabsl:
5569  if (visitUnaryFloatCall(I, ISD::FABS))
5570  return;
5571  break;
5572  case LibFunc::fmin:
5573  case LibFunc::fminf:
5574  case LibFunc::fminl:
5575  if (visitBinaryFloatCall(I, ISD::FMINNUM))
5576  return;
5577  break;
5578  case LibFunc::fmax:
5579  case LibFunc::fmaxf:
5580  case LibFunc::fmaxl:
5581  if (visitBinaryFloatCall(I, ISD::FMAXNUM))
5582  return;
5583  break;
5584  case LibFunc::sin:
5585  case LibFunc::sinf:
5586  case LibFunc::sinl:
5587  if (visitUnaryFloatCall(I, ISD::FSIN))
5588  return;
5589  break;
5590  case LibFunc::cos:
5591  case LibFunc::cosf:
5592  case LibFunc::cosl:
5593  if (visitUnaryFloatCall(I, ISD::FCOS))
5594  return;
5595  break;
5596  case LibFunc::sqrt:
5597  case LibFunc::sqrtf:
5598  case LibFunc::sqrtl:
5599  case LibFunc::sqrt_finite:
5600  case LibFunc::sqrtf_finite:
5601  case LibFunc::sqrtl_finite:
5602  if (visitUnaryFloatCall(I, ISD::FSQRT))
5603  return;
5604  break;
5605  case LibFunc::floor:
5606  case LibFunc::floorf:
5607  case LibFunc::floorl:
5608  if (visitUnaryFloatCall(I, ISD::FFLOOR))
5609  return;
5610  break;
5611  case LibFunc::nearbyint:
5612  case LibFunc::nearbyintf:
5613  case LibFunc::nearbyintl:
5614  if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
5615  return;
5616  break;
5617  case LibFunc::ceil:
5618  case LibFunc::ceilf:
5619  case LibFunc::ceill:
5620  if (visitUnaryFloatCall(I, ISD::FCEIL))
5621  return;
5622  break;
5623  case LibFunc::rint:
5624  case LibFunc::rintf:
5625  case LibFunc::rintl:
5626  if (visitUnaryFloatCall(I, ISD::FRINT))
5627  return;
5628  break;
5629  case LibFunc::round:
5630  case LibFunc::roundf:
5631  case LibFunc::roundl:
5632  if (visitUnaryFloatCall(I, ISD::FROUND))
5633  return;
5634  break;
5635  case LibFunc::trunc:
5636  case LibFunc::truncf:
5637  case LibFunc::truncl:
5638  if (visitUnaryFloatCall(I, ISD::FTRUNC))
5639  return;
5640  break;
5641  case LibFunc::log2:
5642  case LibFunc::log2f:
5643  case LibFunc::log2l:
5644  if (visitUnaryFloatCall(I, ISD::FLOG2))
5645  return;
5646  break;
5647  case LibFunc::exp2:
5648  case LibFunc::exp2f:
5649  case LibFunc::exp2l:
5650  if (visitUnaryFloatCall(I, ISD::FEXP2))
5651  return;
5652  break;
5653  case LibFunc::memcmp:
5654  if (visitMemCmpCall(I))
5655  return;
5656  break;
5657  case LibFunc::memchr:
5658  if (visitMemChrCall(I))
5659  return;
5660  break;
5661  case LibFunc::strcpy:
5662  if (visitStrCpyCall(I, false))
5663  return;
5664  break;
5665  case LibFunc::stpcpy:
5666  if (visitStrCpyCall(I, true))
5667  return;
5668  break;
5669  case LibFunc::strcmp:
5670  if (visitStrCmpCall(I))
5671  return;
5672  break;
5673  case LibFunc::strlen:
5674  if (visitStrLenCall(I))
5675  return;
5676  break;
5677  case LibFunc::strnlen:
5678  if (visitStrNLenCall(I))
5679  return;
5680  break;
5681  }
5682  }
5683  }
5684 
5685  SDValue Callee;
5686  if (!RenameFn)
5687  Callee = getValue(I.getCalledValue());
5688  else
5689  Callee = DAG.getExternalSymbol(
5690  RenameFn,
5692 
5693  // Check if we can potentially perform a tail call. More detailed checking is
5694  // be done within LowerCallTo, after more information about the call is known.
5695  LowerCallTo(&I, Callee, I.isTailCall());
5696 }
5697 
5698 namespace {
5699 
5700 /// AsmOperandInfo - This contains information for each constraint that we are
5701 /// lowering.
5702 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
5703 public:
5704  /// CallOperand - If this is the result output operand or a clobber
5705  /// this is null, otherwise it is the incoming operand to the CallInst.
5706  /// This gets modified as the asm is processed.
5707  SDValue CallOperand;
5708 
5709  /// AssignedRegs - If this is a register or register class operand, this
5710  /// contains the set of register corresponding to the operand.
5711  RegsForValue AssignedRegs;
5712 
5713  explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
5714  : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
5715  }
5716 
5717  /// getCallOperandValEVT - Return the EVT of the Value* that this operand
5718  /// corresponds to. If there is no Value* for this operand, it returns
5719  /// MVT::Other.
5720  EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
5721  const DataLayout &DL) const {
5722  if (!CallOperandVal) return MVT::Other;
5723 
5724  if (isa<BasicBlock>(CallOperandVal))
5725  return TLI.getPointerTy(DL);
5726 
5727  llvm::Type *OpTy = CallOperandVal->getType();
5728 
5729  // FIXME: code duplicated from TargetLowering::ParseConstraints().
5730  // If this is an indirect operand, the operand is a pointer to the
5731  // accessed type.
5732  if (isIndirect) {
5733  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
5734  if (!PtrTy)
5735  report_fatal_error("Indirect operand for inline asm not a pointer!");
5736  OpTy = PtrTy->getElementType();
5737  }
5738 
5739  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
5740  if (StructType *STy = dyn_cast<StructType>(OpTy))
5741  if (STy->getNumElements() == 1)
5742  OpTy = STy->getElementType(0);
5743 
5744  // If OpTy is not a single value, it may be a struct/union that we
5745  // can tile with integers.
5746  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
5747  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
5748  switch (BitSize) {
5749  default: break;
5750  case 1:
5751  case 8:
5752  case 16:
5753  case 32:
5754  case 64:
5755  case 128:
5756  OpTy = IntegerType::get(Context, BitSize);
5757  break;
5758  }
5759  }
5760 
5761  return TLI.getValueType(DL, OpTy, true);
5762  }
5763 };
5764 
5765 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
5766 
5767 } // end anonymous namespace
5768 
5769 /// GetRegistersForValue - Assign registers (virtual or physical) for the
5770 /// specified operand. We prefer to assign virtual registers, to allow the
5771 /// register allocator to handle the assignment process. However, if the asm
5772 /// uses features that we can't model on machineinstrs, we have SDISel do the
5773 /// allocation. This produces generally horrible, but correct, code.
5774 ///
5775 /// OpInfo describes the operand.
5776 ///
5778  const TargetLowering &TLI,
5779  SDLoc DL,
5780  SDISelAsmOperandInfo &OpInfo) {
5781  LLVMContext &Context = *DAG.getContext();
5782 
5783  MachineFunction &MF = DAG.getMachineFunction();
5785 
5786  // If this is a constraint for a single physreg, or a constraint for a
5787  // register class, find it.
5788  std::pair<unsigned, const TargetRegisterClass *> PhysReg =
5790  OpInfo.ConstraintCode,
5791  OpInfo.ConstraintVT);
5792 
5793  unsigned NumRegs = 1;
5794  if (OpInfo.ConstraintVT != MVT::Other) {
5795  // If this is a FP input in an integer register (or visa versa) insert a bit
5796  // cast of the input value. More generally, handle any case where the input
5797  // value disagrees with the register class we plan to stick this in.
5798  if (OpInfo.Type == InlineAsm::isInput &&
5799  PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
5800  // Try to convert to the first EVT that the reg class contains. If the
5801  // types are identical size, use a bitcast to convert (e.g. two differing
5802  // vector types).
5803  MVT RegVT = *PhysReg.second->vt_begin();
5804  if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
5805  OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
5806  RegVT, OpInfo.CallOperand);
5807  OpInfo.ConstraintVT = RegVT;
5808  } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
5809  // If the input is a FP value and we want it in FP registers, do a
5810  // bitcast to the corresponding integer type. This turns an f64 value
5811  // into i64, which can be passed with two i32 values on a 32-bit
5812  // machine.
5813  RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
5814  OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
5815  RegVT, OpInfo.CallOperand);
5816  OpInfo.ConstraintVT = RegVT;
5817  }
5818  }
5819 
5820  NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
5821  }
5822 
5823  MVT RegVT;
5824  EVT ValueVT = OpInfo.ConstraintVT;
5825 
5826  // If this is a constraint for a specific physical register, like {r17},
5827  // assign it now.
5828  if (unsigned AssignedReg = PhysReg.first) {
5829  const TargetRegisterClass *RC = PhysReg.second;
5830  if (OpInfo.ConstraintVT == MVT::Other)
5831  ValueVT = *RC->vt_begin();
5832 
5833  // Get the actual register value type. This is important, because the user
5834  // may have asked for (e.g.) the AX register in i32 type. We need to
5835  // remember that AX is actually i16 to get the right extension.
5836  RegVT = *RC->vt_begin();
5837 
5838  // This is a explicit reference to a physical register.
5839  Regs.push_back(AssignedReg);
5840 
5841  // If this is an expanded reference, add the rest of the regs to Regs.
5842  if (NumRegs != 1) {
5844  for (; *I != AssignedReg; ++I)
5845  assert(I != RC->end() && "Didn't find reg!");
5846 
5847  // Already added the first reg.
5848  --NumRegs; ++I;
5849  for (; NumRegs; --NumRegs, ++I) {
5850  assert(I != RC->end() && "Ran out of registers to allocate!");
5851  Regs.push_back(*I);
5852  }
5853  }
5854 
5855  OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
5856  return;
5857  }
5858 
5859  // Otherwise, if this was a reference to an LLVM register class, create vregs
5860  // for this reference.
5861  if (const TargetRegisterClass *RC = PhysReg.second) {
5862  RegVT = *RC->vt_begin();
5863  if (OpInfo.ConstraintVT == MVT::Other)
5864  ValueVT = RegVT;
5865 
5866  // Create the appropriate number of virtual registers.
5867  MachineRegisterInfo &RegInfo = MF.getRegInfo();
5868  for (; NumRegs; --NumRegs)
5869  Regs.push_back(RegInfo.createVirtualRegister(RC));
5870 
5871  OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
5872  return;
5873  }
5874 
5875  // Otherwise, we couldn't allocate enough registers for this.
5876 }
5877 
5878 /// visitInlineAsm - Handle a call to an InlineAsm object.
5879 ///
5880 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
5881  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
5882 
5883  /// ConstraintOperands - Information about all of the constraints.
5884  SDISelAsmOperandInfoVector ConstraintOperands;
5885 
5886  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5889 
5890  bool hasMemory = false;
5891 
5892  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
5893  unsigned ResNo = 0; // ResNo - The result number of the next output.
5894  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
5895  ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
5896  SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
5897 
5898  MVT OpVT = MVT::Other;
5899 
5900  // Compute the value type for each operand.
5901  switch (OpInfo.Type) {
5902  case InlineAsm::isOutput:
5903  // Indirect outputs just consume an argument.
5904  if (OpInfo.isIndirect) {
5905  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
5906  break;
5907  }
5908 
5909  // The return value of the call is this value. As such, there is no
5910  // corresponding argument.
5911  assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
5912  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
5913  OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
5914  STy->getElementType(ResNo));
5915  } else {
5916  assert(ResNo == 0 && "Asm only has one result!");
5917  OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
5918  }
5919  ++ResNo;
5920  break;
5921  case InlineAsm::isInput:
5922  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
5923  break;
5924  case InlineAsm::isClobber:
5925  // Nothing to do.
5926  break;
5927  }
5928 
5929  // If this is an input or an indirect output, process the call argument.
5930  // BasicBlocks are labels, currently appearing only in asm's.
5931  if (OpInfo.CallOperandVal) {
5932  if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
5933  OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
5934  } else {
5935  OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
5936  }
5937 
5938  OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI,
5939  DAG.getDataLayout()).getSimpleVT();
5940  }
5941 
5942  OpInfo.ConstraintVT = OpVT;
5943 
5944  // Indirect operand accesses access memory.
5945  if (OpInfo.isIndirect)
5946  hasMemory = true;
5947  else {
5948  for (unsigned j = 0, ee = OpInfo.Codes.size(); j != ee; ++j) {
5950  CType = TLI.getConstraintType(OpInfo.Codes[j]);
5951  if (CType == TargetLowering::C_Memory) {
5952  hasMemory = true;
5953  break;
5954  }
5955  }
5956  }
5957  }
5958 
5959  SDValue Chain, Flag;
5960 
5961  // We won't need to flush pending loads if this asm doesn't touch
5962  // memory and is nonvolatile.
5963  if (hasMemory || IA->hasSideEffects())
5964  Chain = getRoot();
5965  else
5966  Chain = DAG.getRoot();
5967 
5968  // Second pass over the constraints: compute which constraint option to use
5969  // and assign registers to constraints that want a specific physreg.
5970  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
5971  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
5972 
5973  // If this is an output operand with a matching input operand, look up the
5974  // matching input. If their types mismatch, e.g. one is an integer, the
5975  // other is floating point, or their sizes are different, flag it as an
5976  // error.
5977  if (OpInfo.hasMatchingInput()) {
5978  SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
5979 
5980  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
5982  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
5983  TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
5984  OpInfo.ConstraintVT);
5985  std::pair<unsigned, const TargetRegisterClass *> InputRC =
5986  TLI.getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
5987  Input.ConstraintVT);
5988  if ((OpInfo.ConstraintVT.isInteger() !=
5989  Input.ConstraintVT.isInteger()) ||
5990  (MatchRC.second != InputRC.second)) {
5991  report_fatal_error("Unsupported asm: input constraint"
5992  " with a matching output constraint of"
5993  " incompatible type!");
5994  }
5995  Input.ConstraintVT = OpInfo.ConstraintVT;
5996  }
5997  }
5998 
5999  // Compute the constraint code and ConstraintType to use.
6000  TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
6001 
6002  if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6003  OpInfo.Type == InlineAsm::isClobber)
6004  continue;
6005 
6006  // If this is a memory input, and if the operand is not indirect, do what we
6007  // need to to provide an address for the memory input.
6008  if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6009  !OpInfo.isIndirect) {
6010  assert((OpInfo.isMultipleAlternative ||
6011  (OpInfo.Type == InlineAsm::isInput)) &&
6012  "Can only indirectify direct input operands!");
6013 
6014  // Memory operands really want the address of the value. If we don't have
6015  // an indirect input, put it in the constpool if we can, otherwise spill
6016  // it to a stack slot.
6017  // TODO: This isn't quite right. We need to handle these according to
6018  // the addressing mode that the constraint wants. Also, this may take
6019  // an additional register for the computation and we don't want that
6020  // either.
6021 
6022  // If the operand is a float, integer, or vector constant, spill to a
6023  // constant pool entry to get its address.
6024  const Value *OpVal = OpInfo.CallOperandVal;
6025  if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6026  isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6027  OpInfo.CallOperand = DAG.getConstantPool(
6028  cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6029  } else {
6030  // Otherwise, create a stack slot and emit a store to it before the
6031  // asm.
6032  Type *Ty = OpVal->getType();
6033  auto &DL = DAG.getDataLayout();
6034  uint64_t TySize = DL.getTypeAllocSize(Ty);
6035  unsigned Align = DL.getPrefTypeAlignment(Ty);
6037  int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
6038  SDValue StackSlot =
6040  Chain = DAG.getStore(Chain, getCurSDLoc(),
6041  OpInfo.CallOperand, StackSlot,
6043  false, false, 0);
6044  OpInfo.CallOperand = StackSlot;
6045  }
6046 
6047  // There is no longer a Value* corresponding to this operand.
6048  OpInfo.CallOperandVal = nullptr;
6049 
6050  // It is now an indirect operand.
6051  OpInfo.isIndirect = true;
6052  }
6053 
6054  // If this constraint is for a specific register, allocate it before
6055  // anything else.
6056  if (OpInfo.ConstraintType == TargetLowering::C_Register)
6057  GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6058  }
6059 
6060  // Second pass - Loop over all of the operands, assigning virtual or physregs
6061  // to register class operands.
6062  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6063  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6064 
6065  // C_Register operands have already been allocated, Other/Memory don't need
6066  // to be.
6067  if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
6068  GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6069  }
6070 
6071  // AsmNodeOperands - The operands for the ISD::INLINEASM node.
6072  std::vector<SDValue> AsmNodeOperands;
6073  AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
6074  AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
6075  IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
6076 
6077  // If we have a !srcloc metadata node associated with it, we want to attach
6078  // this to the ultimately generated inline asm machineinstr. To do this, we
6079  // pass in the third operand as this (potentially null) inline asm MDNode.
6080  const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
6081  AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
6082 
6083  // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6084  // bits as operand 3.
6085  unsigned ExtraInfo = 0;
6086  if (IA->hasSideEffects())
6087  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
6088  if (IA->isAlignStack())
6089  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
6090  // Set the asm dialect.
6091  ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
6092 
6093  // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
6094  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
6095  TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
6096 
6097  // Compute the constraint code and ConstraintType to use.
6098  TLI.ComputeConstraintToUse(OpInfo, SDValue());
6099 
6100  // Ideally, we would only check against memory constraints. However, the
6101  // meaning of an other constraint can be target-specific and we can't easily
6102  // reason about it. Therefore, be conservative and set MayLoad/MayStore
6103  // for other constriants as well.
6104  if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
6106  if (OpInfo.Type == InlineAsm::isInput)
6107  ExtraInfo |= InlineAsm::Extra_MayLoad;
6108  else if (OpInfo.Type == InlineAsm::isOutput)
6109  ExtraInfo |= InlineAsm::Extra_MayStore;
6110  else if (OpInfo.Type == InlineAsm::isClobber)
6112  }
6113  }
6114 
6115  AsmNodeOperands.push_back(DAG.getTargetConstant(
6116  ExtraInfo, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6117 
6118  // Loop over all of the inputs, copying the operand values into the
6119  // appropriate registers and processing the output regs.
6120  RegsForValue RetValRegs;
6121 
6122  // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
6123  std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
6124 
6125  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6126  SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6127 
6128  switch (OpInfo.Type) {
6129  case InlineAsm::isOutput: {
6130  if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
6131  OpInfo.ConstraintType != TargetLowering::C_Register) {
6132  // Memory output, or 'other' output (e.g. 'X' constraint).
6133  assert(OpInfo.isIndirect && "Memory output must be indirect operand");
6134 
6135  unsigned ConstraintID =
6136  TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
6137  assert(ConstraintID != InlineAsm::Constraint_Unknown &&
6138  "Failed to convert memory constraint code to constraint id.");
6139 
6140  // Add information to the INLINEASM node to know about this output.
6141  unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
6142  OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
6143  AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
6144  MVT::i32));
6145  AsmNodeOperands.push_back(OpInfo.CallOperand);
6146  break;
6147  }
6148 
6149  // Otherwise, this is a register or register class output.
6150 
6151  // Copy the output from the appropriate register. Find a register that
6152  // we can use.
6153  if (OpInfo.AssignedRegs.Regs.empty()) {
6154  LLVMContext &Ctx = *DAG.getContext();
6155  Ctx.emitError(CS.getInstruction(),
6156  "couldn't allocate output register for constraint '" +
6157  Twine(OpInfo.ConstraintCode) + "'");
6158  return;
6159  }
6160 
6161  // If this is an indirect operand, store through the pointer after the
6162  // asm.
6163  if (OpInfo.isIndirect) {
6164  IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
6165  OpInfo.CallOperandVal));
6166  } else {
6167  // This is the result value of the call.
6168  assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
6169  // Concatenate this output onto the outputs list.
6170  RetValRegs.append(OpInfo.AssignedRegs);
6171  }
6172 
6173  // Add information to the INLINEASM node to know that this register is
6174  // set.
6175  OpInfo.AssignedRegs
6176  .AddInlineAsmOperands(OpInfo.isEarlyClobber
6179  false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
6180  break;
6181  }
6182  case InlineAsm::isInput: {
6183  SDValue InOperandVal = OpInfo.CallOperand;
6184 
6185  if (OpInfo.isMatchingInputConstraint()) { // Matching constraint?
6186  // If this is required to match an output register we have already set,
6187  // just use its register.
6188  unsigned OperandNo = OpInfo.getMatchedOperand();
6189 
6190  // Scan until we find the definition we already emitted of this operand.
6191  // When we find it, create a RegsForValue operand.
6192  unsigned CurOp = InlineAsm::Op_FirstOperand;
6193  for (; OperandNo; --OperandNo) {
6194  // Advance to the next operand.
6195  unsigned OpFlag =
6196  cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6197  assert((InlineAsm::isRegDefKind(OpFlag) ||
6199  InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?");
6200  CurOp += InlineAsm::getNumOperandRegisters(OpFlag)+1;
6201  }
6202 
6203  unsigned OpFlag =
6204  cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6205  if (InlineAsm::isRegDefKind(OpFlag) ||
6207  // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
6208  if (OpInfo.isIndirect) {
6209  // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
6210  LLVMContext &Ctx = *DAG.getContext();
6211  Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:"
6212  " don't know how to handle tied "
6213  "indirect register inputs");
6214  return;
6215  }
6216 
6217  RegsForValue MatchedRegs;
6218  MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType());
6219  MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
6220  MatchedRegs.RegVTs.push_back(RegVT);
6222  for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag);
6223  i != e; ++i) {
6224  if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
6225  MatchedRegs.Regs.push_back(RegInfo.createVirtualRegister(RC));
6226  else {
6227  LLVMContext &Ctx = *DAG.getContext();
6228  Ctx.emitError(CS.getInstruction(),
6229  "inline asm error: This value"
6230  " type register class is not natively supported!");
6231  return;
6232  }
6233  }
6234  SDLoc dl = getCurSDLoc();
6235  // Use the produced MatchedRegs object to
6236  MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
6237  Chain, &Flag, CS.getInstruction());
6239  true, OpInfo.getMatchedOperand(), dl,
6240  DAG, AsmNodeOperands);
6241  break;
6242  }
6243 
6244  assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
6245  assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
6246  "Unexpected number of operands");
6247  // Add information to the INLINEASM node to know about this input.
6248  // See InlineAsm.h isUseOperandTiedToDef.
6250  OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
6251  OpInfo.getMatchedOperand());
6252  AsmNodeOperands.push_back(DAG.getTargetConstant(
6253  OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6254  AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
6255  break;
6256  }
6257 
6258  // Treat indirect 'X' constraint as memory.
6259  if (OpInfo.ConstraintType == TargetLowering::C_Other &&
6260  OpInfo.isIndirect)
6261  OpInfo.ConstraintType = TargetLowering::C_Memory;
6262 
6263  if (OpInfo.ConstraintType == TargetLowering::C_Other) {
6264  std::vector<SDValue> Ops;
6265  TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
6266  Ops, DAG);
6267  if (Ops.empty()) {
6268  LLVMContext &Ctx = *DAG.getContext();
6269  Ctx.emitError(CS.getInstruction(),
6270  "invalid operand for inline asm constraint '" +
6271  Twine(OpInfo.ConstraintCode) + "'");
6272  return;
6273  }
6274 
6275  // Add information to the INLINEASM node to know about this input.
6276  unsigned ResOpType =
6278  AsmNodeOperands.push_back(DAG.getTargetConstant(
6279  ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6280  AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
6281  break;
6282  }
6283 
6284  if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
6285  assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
6286  assert(InOperandVal.getValueType() ==
6287  TLI.getPointerTy(DAG.getDataLayout()) &&
6288  "Memory operands expect pointer values");
6289 
6290  unsigned ConstraintID =
6291  TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
6292  assert(ConstraintID != InlineAsm::Constraint_Unknown &&
6293  "Failed to convert memory constraint code to constraint id.");
6294 
6295  // Add information to the INLINEASM node to know about this input.
6296  unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
6297  ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
6298  AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
6299  getCurSDLoc(),
6300  MVT::i32));
6301  AsmNodeOperands.push_back(InOperandVal);
6302  break;
6303  }
6304 
6305  assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
6306  OpInfo.ConstraintType == TargetLowering::C_Register) &&
6307  "Unknown constraint type!");
6308 
6309  // TODO: Support this.
6310  if (OpInfo.isIndirect) {
6311  LLVMContext &Ctx = *DAG.getContext();
6312  Ctx.emitError(CS.getInstruction(),
6313  "Don't know how to handle indirect register inputs yet "
6314  "for constraint '" +
6315  Twine(OpInfo.ConstraintCode) + "'");
6316  return;
6317  }
6318 
6319  // Copy the input into the appropriate registers.
6320  if (OpInfo.AssignedRegs.Regs.empty()) {
6321  LLVMContext &Ctx = *DAG.getContext();
6322  Ctx.emitError(CS.getInstruction(),
6323  "couldn't allocate input reg for constraint '" +
6324  Twine(OpInfo.ConstraintCode) + "'");
6325  return;
6326  }
6327 
6328  SDLoc dl = getCurSDLoc();
6329 
6330  OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
6331  Chain, &Flag, CS.getInstruction());
6332 
6333  OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
6334  dl, DAG, AsmNodeOperands);
6335  break;
6336  }
6337  case InlineAsm::isClobber: {
6338  // Add the clobbered value to the operand list, so that the register
6339  // allocator is aware that the physreg got clobbered.
6340  if (!OpInfo.AssignedRegs.Regs.empty())
6341  OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
6342  false, 0, getCurSDLoc(), DAG,
6343  AsmNodeOperands);
6344  break;
6345  }
6346  }
6347  }
6348 
6349  // Finish up input operands. Set the input chain and add the flag last.
6350  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
6351  if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
6352 
6354  DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
6355  Flag = Chain.getValue(1);
6356 
6357  // If this asm returns a register value, copy the result from that register
6358  // and set it as the value of the call.
6359  if (!RetValRegs.Regs.empty()) {
6360  SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
6361  Chain, &Flag, CS.getInstruction());
6362 
6363  // FIXME: Why don't we do this for inline asms with MRVs?
6364  if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
6365  EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
6366 
6367  // If any of the results of the inline asm is a vector, it may have the
6368  // wrong width/num elts. This can happen for register classes that can
6369  // contain multiple different value types. The preg or vreg allocated may
6370  // not have the same VT as was expected. Convert it to the right type
6371  // with bit_convert.
6372  if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
6374  ResultType, Val);
6375 
6376  } else if (ResultType != Val.getValueType() &&
6377  ResultType.isInteger() && Val.getValueType().isInteger()) {
6378  // If a result value was tied to an input value, the computed result may
6379  // have a wider width than the expected result. Extract the relevant
6380  // portion.
6381  Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
6382  }
6383 
6384  assert(ResultType == Val.getValueType() && "Asm result value mismatch!");
6385  }
6386 
6387  setValue(CS.getInstruction(), Val);
6388  // Don't need to use this as a chain in this case.
6389  if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
6390  return;
6391  }
6392 
6393  std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
6394 
6395  // Process indirect outputs, first output all of the flagged copies out of
6396  // physregs.
6397  for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
6398  RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
6399  const Value *Ptr = IndirectStoresToEmit[i].second;
6400  SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
6401  Chain, &Flag, IA);
6402  StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
6403  }
6404 
6405  // Emit the non-flagged stores from the physregs.
6406  SmallVector<SDValue, 8> OutChains;
6407  for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
6408  SDValue Val = DAG.getStore(Chain, getCurSDLoc(),
6409  StoresToEmit[i].first,
6410  getValue(StoresToEmit[i].second),
6411  MachinePointerInfo(StoresToEmit[i].second),
6412  false, false, 0);
6413  OutChains.push_back(Val);
6414  }
6415 
6416  if (!OutChains.empty())
6417  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
6418 
6419  DAG.setRoot(Chain);
6420 }
6421 
6422 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
6424  MVT::Other, getRoot(),
6425  getValue(I.getArgOperand(0)),
6426  DAG.getSrcValue(I.getArgOperand(0))));
6427 }
6428 
6429 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
6430  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6431  const DataLayout &DL = DAG.getDataLayout();
6433  getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
6434  DAG.getSrcValue(I.getOperand(0)),
6435  DL.getABITypeAlignment(I.getType()));
6436  setValue(&I, V);
6437  DAG.setRoot(V.getValue(1));
6438 }
6439 
6440 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
6442  MVT::Other, getRoot(),
6443  getValue(I.getArgOperand(0)),
6444  DAG.getSrcValue(I.getArgOperand(0))));
6445 }
6446 
6447 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
6449  MVT::Other, getRoot(),
6450  getValue(I.getArgOperand(0)),
6451  getValue(I.getArgOperand(1)),
6453  DAG.getSrcValue(I.getArgOperand(1))));
6454 }
6455 
6456 /// \brief Lower an argument list according to the target calling convention.
6457 ///
6458 /// \return A tuple of <return-value, token-chain>
6459 ///
6460 /// This is a helper for lowering intrinsics that follow a target calling
6461 /// convention or require stack pointer adjustment. Only a subset of the
6462 /// intrinsic's operands need to participate in the calling convention.
6463 std::pair<SDValue, SDValue>
6465  unsigned NumArgs, SDValue Callee,
6466  Type *ReturnTy,
6467  MachineBasicBlock *LandingPad,
6468  bool IsPatchPoint) {
6470  Args.reserve(NumArgs);
6471 
6472  // Populate the argument list.
6473  // Attributes for args start at offset 1, after the return attribute.
6474  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1;
6475  ArgI != ArgE; ++ArgI) {
6476  const Value *V = CS->getOperand(ArgI);
6477 
6478  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
6479 
6481  Entry.Node = getValue(V);
6482  Entry.Ty = V->getType();
6483  Entry.setAttributes(&CS, AttrI);
6484  Args.push_back(Entry);
6485  }
6486 
6489  .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args), NumArgs)
6490  .setDiscardResult(CS->use_empty()).setIsPatchPoint(IsPatchPoint);
6491 
6492  return lowerInvokable(CLI, LandingPad);
6493 }
6494 
6495 /// \brief Add a stack map intrinsic call's live variable operands to a stackmap
6496 /// or patchpoint target node's operand list.
6497 ///
6498 /// Constants are converted to TargetConstants purely as an optimization to
6499 /// avoid constant materialization and register allocation.
6500 ///
6501 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
6502 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
6503 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
6504 /// address materialization and register allocation, but may also be required
6505 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
6506 /// alloca in the entry block, then the runtime may assume that the alloca's
6507 /// StackMap location can be read immediately after compilation and that the
6508 /// location is valid at any point during execution (this is similar to the
6509 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
6510 /// only available in a register, then the runtime would need to trap when
6511 /// execution reaches the StackMap in order to read the alloca's location.
6512 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
6514  SelectionDAGBuilder &Builder) {
6515  for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
6516  SDValue OpVal = Builder.getValue(CS.getArgument(i));
6517  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
6518  Ops.push_back(
6520  Ops.push_back(
6521  Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
6522  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
6523  const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
6524  Ops.push_back(Builder.DAG.getTargetFrameIndex(
6525  FI->getIndex(), TLI.getPointerTy(Builder.DAG.getDataLayout())));
6526  } else
6527  Ops.push_back(OpVal);
6528  }
6529 }
6530 
6531 /// \brief Lower llvm.experimental.stackmap directly to its target opcode.
6532 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
6533  // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
6534  // [live variables...])
6535 
6536  assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
6537 
6538  SDValue Chain, InFlag, Callee, NullPtr;
6540 
6541  SDLoc DL = getCurSDLoc();
6542  Callee = getValue(CI.getCalledValue());
6543  NullPtr = DAG.getIntPtrConstant(0, DL, true);
6544 
6545  // The stackmap intrinsic only records the live variables (the arguemnts
6546  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
6547  // intrinsic, this won't be lowered to a function call. This means we don't
6548  // have to worry about calling conventions and target specific lowering code.
6549  // Instead we perform the call lowering right here.
6550  //
6551  // chain, flag = CALLSEQ_START(chain, 0)
6552  // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
6553  // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
6554  //
6555  Chain = DAG.getCALLSEQ_START(getRoot(), NullPtr, DL);
6556  InFlag = Chain.getValue(1);
6557 
6558  // Add the <id> and <numBytes> constants.
6561  cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
6564  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
6565  MVT::i32));
6566 
6567  // Push live variables for the stack map.
6568  addStackMapLiveVars(&CI, 2, DL, Ops, *this);
6569 
6570  // We are not pushing any register mask info here on the operands list,
6571  // because the stackmap doesn't clobber anything.
6572 
6573  // Push the chain and the glue flag.
6574  Ops.push_back(Chain);
6575  Ops.push_back(InFlag);
6576 
6577  // Create the STACKMAP node.
6579  SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
6580  Chain = SDValue(SM, 0);
6581  InFlag = Chain.getValue(1);
6582 
6583  Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
6584 
6585  // Stackmaps don't generate values, so nothing goes into the NodeMap.
6586 
6587  // Set the root to the target-lowered call chain.
6588  DAG.setRoot(Chain);
6589 
6590  // Inform the Frame Information that we have a stackmap in this function.
6592 }
6593 
6594 /// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
6595 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
6596  MachineBasicBlock *LandingPad) {
6597  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
6598  // i32 <numBytes>,
6599  // i8* <target>,
6600  // i32 <numArgs>,
6601  // [Args...],
6602  // [live variables...])
6603 
6604  CallingConv::ID CC = CS.getCallingConv();
6605  bool IsAnyRegCC = CC == CallingConv::AnyReg;
6606  bool HasDef = !CS->getType()->isVoidTy();
6607  SDLoc dl = getCurSDLoc();
6608  SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
6609 
6610  // Handle immediate and symbolic callees.
6611  if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
6612  Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
6613  /*isTarget=*/true);
6614  else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
6615  Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
6616  SDLoc(SymbolicCallee),
6617  SymbolicCallee->getValueType(0));
6618 
6619  // Get the real number of arguments participating in the call <numArgs>
6621  unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
6622 
6623  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
6624  // Intrinsics include all meta-operands up to but not including CC.
6625  unsigned NumMetaOpers = PatchPointOpers::CCPos;
6626  assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
6627  "Not enough arguments provided to the patchpoint intrinsic");
6628 
6629  // For AnyRegCC the arguments are lowered later on manually.
6630  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
6631  Type *ReturnTy =
6632  IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
6633  std::pair<SDValue, SDValue> Result =
6634  lowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
6635  LandingPad, true);
6636 
6637  SDNode *CallEnd = Result.second.getNode();
6638  if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
6639  CallEnd = CallEnd->getOperand(0).getNode();
6640 
6641  /// Get a call instruction from the call sequence chain.
6642  /// Tail calls are not allowed.
6643  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
6644  "Expected a callseq node.");
6645  SDNode *Call = CallEnd->getOperand(0).getNode();
6646  bool HasGlue = Call->getGluedNode();
6647 
6648  // Replace the target specific call node with the patchable intrinsic.
6650 
6651  // Add the <id> and <numBytes> constants.
6652  SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
6654  cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
6655  SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
6657  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
6658  MVT::i32));
6659 
6660  // Add the callee.
6661  Ops.push_back(Callee);
6662 
6663  // Adjust <numArgs> to account for any arguments that have been passed on the
6664  // stack instead.
6665  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
6666  unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
6667  NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
6668  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
6669 
6670  // Add the calling convention
6671  Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
6672 
6673  // Add the arguments we omitted previously. The register allocator should
6674  // place these in any free register.
6675  if (IsAnyRegCC)
6676  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
6677  Ops.push_back(getValue(CS.getArgument(i)));
6678 
6679  // Push the arguments from the call instruction up to the register mask.
6680  SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
6681  Ops.append(Call->op_begin() + 2, e);
6682 
6683  // Push live variables for the stack map.
6684  addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
6685 
6686  // Push the register mask info.
6687  if (HasGlue)
6688  Ops.push_back(*(Call->op_end()-2));
6689  else
6690  Ops.push_back(*(Call->op_end()-1));
6691 
6692  // Push the chain (this is originally the first operand of the call, but
6693  // becomes now the last or second to last operand).
6694  Ops.push_back(*(Call->op_begin()));
6695 
6696  // Push the glue flag (last operand).
6697  if (HasGlue)
6698  Ops.push_back(*(Call->op_end()-1));
6699 
6700  SDVTList NodeTys;
6701  if (IsAnyRegCC && HasDef) {
6702  // Create the return types based on the intrinsic definition
6703  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6704  SmallVector<EVT, 3> ValueVTs;
6705  ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
6706  assert(ValueVTs.size() == 1 && "Expected only one return value type.");
6707 
6708  // There is always a chain and a glue type at the end
6709  ValueVTs.push_back(MVT::Other);
6710  ValueVTs.push_back(MVT::Glue);
6711  NodeTys = DAG.getVTList(ValueVTs);
6712  } else
6713  NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6714 
6715  // Replace the target specific call node with a PATCHPOINT node.
6717  dl, NodeTys, Ops);
6718 
6719  // Update the NodeMap.
6720  if (HasDef) {
6721  if (IsAnyRegCC)
6722  setValue(CS.getInstruction(), SDValue(MN, 0));
6723  else
6724  setValue(CS.getInstruction(), Result.first);
6725  }
6726 
6727  // Fixup the consumers of the intrinsic. The chain and glue may be used in the
6728  // call sequence. Furthermore the location of the chain and glue can change
6729  // when the AnyReg calling convention is used and the intrinsic returns a
6730  // value.
6731  if (IsAnyRegCC && HasDef) {
6732  SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
6733  SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
6734  DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
6735  } else
6736  DAG.ReplaceAllUsesWith(Call, MN);
6737  DAG.DeleteNode(Call);
6738 
6739  // Inform the Frame Information that we have a patchpoint in this function.
6741 }
6742 
6743 /// Returns an AttributeSet representing the attributes applied to the return
6744 /// value of the given call.
6747  if (CLI.RetSExt)
6748  Attrs.push_back(Attribute::SExt);
6749  if (CLI.RetZExt)
6750  Attrs.push_back(Attribute::ZExt);
6751  if (CLI.IsInReg)
6752  Attrs.push_back(Attribute::InReg);
6753 
6754  return AttributeSet::get(CLI.RetTy->getContext(), AttributeSet::ReturnIndex,
6755  Attrs);
6756 }
6757 
6758 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
6759 /// implementation, which just calls LowerCall.
6760 /// FIXME: When all targets are
6761 /// migrated to using LowerCall, this hook should be integrated into SDISel.
6762 std::pair<SDValue, SDValue>
6764  // Handle the incoming return values from the call.
6765  CLI.Ins.clear();
6766  Type *OrigRetTy = CLI.RetTy;
6767  SmallVector<EVT, 4> RetTys;
6769  auto &DL = CLI.DAG.getDataLayout();
6770  ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
6771 
6773  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
6774 
6775  bool CanLowerReturn =
6776  this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
6777  CLI.IsVarArg, Outs, CLI.RetTy->getContext());
6778 
6779  SDValue DemoteStackSlot;
6780  int DemoteStackIdx = -100;
6781  if (!CanLowerReturn) {
6782  // FIXME: equivalent assert?
6783  // assert(!CS.hasInAllocaArgument() &&
6784  // "sret demotion is incompatible with inalloca");
6785  uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
6786  unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
6788  DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
6789  Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
6790 
6791  DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy(DL));
6792  ArgListEntry Entry;
6793  Entry.Node = DemoteStackSlot;
6794  Entry.Ty = StackSlotPtrType;
6795  Entry.isSExt = false;
6796  Entry.isZExt = false;
6797  Entry.isInReg = false;
6798  Entry.isSRet = true;
6799  Entry.isNest = false;
6800  Entry.isByVal = false;
6801  Entry.isReturned = false;
6802  Entry.Alignment = Align;
6803  CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
6804  CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
6805 
6806  // sret demotion isn't compatible with tail-calls, since the sret argument
6807  // points into the callers stack frame.
6808  CLI.IsTailCall = false;
6809  } else {
6810  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
6811  EVT VT = RetTys[I];
6812  MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
6813  unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
6814  for (unsigned i = 0; i != NumRegs; ++i) {
6815  ISD::InputArg MyFlags;
6816  MyFlags.VT = RegisterVT;
6817  MyFlags.ArgVT = VT;
6818  MyFlags.Used = CLI.IsReturnValueUsed;
6819  if (CLI.RetSExt)
6820  MyFlags.Flags.setSExt();
6821  if (CLI.RetZExt)
6822  MyFlags.Flags.setZExt();
6823  if (CLI.IsInReg)
6824  MyFlags.Flags.setInReg();
6825  CLI.Ins.push_back(MyFlags);
6826  }
6827  }
6828  }
6829 
6830  // Handle all of the outgoing arguments.
6831  CLI.Outs.clear();
6832  CLI.OutVals.clear();
6833  ArgListTy &Args = CLI.getArgs();
6834  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
6835  SmallVector<EVT, 4> ValueVTs;
6836  ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
6837  Type *FinalType = Args[i].Ty;
6838  if (Args[i].isByVal)
6839  FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
6840  bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
6841  FinalType, CLI.CallConv, CLI.IsVarArg);
6842  for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
6843  ++Value) {
6844  EVT VT = ValueVTs[Value];
6845  Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
6846  SDValue Op = SDValue(Args[i].Node.getNode(),
6847  Args[i].Node.getResNo() + Value);
6849  unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
6850 
6851  if (Args[i].isZExt)
6852  Flags.setZExt();
6853  if (Args[i].isSExt)
6854  Flags.setSExt();
6855  if (Args[i].isInReg)
6856  Flags.setInReg();
6857  if (Args[i].isSRet)
6858  Flags.setSRet();
6859  if (Args[i].isByVal)
6860  Flags.setByVal();
6861  if (Args[i].isInAlloca) {
6862  Flags.setInAlloca();
6863  // Set the byval flag for CCAssignFn callbacks that don't know about
6864  // inalloca. This way we can know how many bytes we should've allocated
6865  // and how many bytes a callee cleanup function will pop. If we port
6866  // inalloca to more targets, we'll have to add custom inalloca handling
6867  // in the various CC lowering callbacks.
6868  Flags.setByVal();
6869  }
6870  if (Args[i].isByVal || Args[i].isInAlloca) {
6871  PointerType *Ty = cast<PointerType>(Args[i].Ty);
6872  Type *ElementTy = Ty->getElementType();
6873  Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
6874  // For ByVal, alignment should come from FE. BE will guess if this
6875  // info is not there but there are cases it cannot get right.
6876  unsigned FrameAlign;
6877  if (Args[i].Alignment)
6878  FrameAlign = Args[i].Alignment;
6879  else
6880  FrameAlign = getByValTypeAlignment(ElementTy, DL);
6881  Flags.setByValAlign(FrameAlign);
6882  }
6883  if (Args[i].isNest)
6884  Flags.setNest();
6885  if (NeedsRegBlock)
6886  Flags.setInConsecutiveRegs();
6887  Flags.setOrigAlign(OriginalAlignment);
6888 
6889  MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT);
6890  unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT);
6891  SmallVector<SDValue, 4> Parts(NumParts);
6892  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
6893 
6894  if (Args[i].isSExt)
6895  ExtendKind = ISD::SIGN_EXTEND;
6896  else if (Args[i].isZExt)
6897  ExtendKind = ISD::ZERO_EXTEND;
6898 
6899  // Conservatively only handle 'returned' on non-vectors for now
6900  if (Args[i].isReturned && !Op.getValueType().isVector()) {
6901  assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
6902  "unexpected use of 'returned'");
6903  // Before passing 'returned' to the target lowering code, ensure that
6904  // either the register MVT and the actual EVT are the same size or that
6905  // the return value and argument are extended in the same way; in these
6906  // cases it's safe to pass the argument register value unchanged as the
6907  // return register value (although it's at the target's option whether
6908  // to do so)
6909  // TODO: allow code generation to take advantage of partially preserved
6910  // registers rather than clobbering the entire register when the
6911  // parameter extension method is not compatible with the return
6912  // extension method
6913  if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
6914  (ExtendKind != ISD::ANY_EXTEND &&
6915  CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt))
6916  Flags.setReturned();
6917  }
6918 
6919  getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
6920  CLI.CS ? CLI.CS->getInstruction() : nullptr, ExtendKind);
6921 
6922  for (unsigned j = 0; j != NumParts; ++j) {
6923  // if it isn't first piece, alignment must be 1
6924  ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
6925  i < CLI.NumFixedArgs,
6926  i, j*Parts[j].getValueType().getStoreSize());
6927  if (NumParts > 1 && j == 0)
6928  MyFlags.Flags.setSplit();
6929  else if (j != 0)
6930  MyFlags.Flags.setOrigAlign(1);
6931 
6932  CLI.Outs.push_back(MyFlags);
6933  CLI.OutVals.push_back(Parts[j]);
6934  }
6935 
6936  if (NeedsRegBlock && Value == NumValues - 1)
6937  CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
6938  }
6939  }
6940 
6941  SmallVector<SDValue, 4> InVals;
6942  CLI.Chain = LowerCall(CLI, InVals);
6943 
6944  // Verify that the target's LowerCall behaved as expected.
6945  assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
6946  "LowerCall didn't return a valid chain!");
6947  assert((!CLI.IsTailCall || InVals.empty()) &&
6948  "LowerCall emitted a return value for a tail call!");
6949  assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
6950  "LowerCall didn't emit the correct number of values!");
6951 
6952  // For a tail call, the return value is merely live-out and there aren't
6953  // any nodes in the DAG representing it. Return a special value to
6954  // indicate that a tail call has been emitted and no more Instructions
6955  // should be processed in the current block.
6956  if (CLI.IsTailCall) {
6957  CLI.DAG.setRoot(CLI.Chain);
6958  return std::make_pair(SDValue(), SDValue());
6959  }
6960 
6961  DEBUG(for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
6962  assert(InVals[i].getNode() &&
6963  "LowerCall emitted a null value!");
6964  assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
6965  "LowerCall emitted a value with the wrong type!");
6966  });
6967 
6968  SmallVector<SDValue, 4> ReturnValues;
6969  if (!CanLowerReturn) {
6970  // The instruction result is the result of loading from the
6971  // hidden sret parameter.
6972  SmallVector<EVT, 1> PVTs;
6973  Type *PtrRetTy = PointerType::getUnqual(OrigRetTy);
6974 
6975  ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
6976  assert(PVTs.size() == 1 && "Pointers should fit in one register");
6977  EVT PtrVT = PVTs[0];
6978 
6979  unsigned NumValues = RetTys.size();
6980  ReturnValues.resize(NumValues);
6981  SmallVector<SDValue, 4> Chains(NumValues);
6982 
6983  for (unsigned i = 0; i < NumValues; ++i) {
6984  SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
6985  CLI.DAG.getConstant(Offsets[i], CLI.DL,
6986  PtrVT));
6987  SDValue L = CLI.DAG.getLoad(
6988  RetTys[i], CLI.DL, CLI.Chain, Add,
6989  MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]), false,
6990  false, false, 1);
6991  ReturnValues[i] = L;
6992  Chains[i] = L.getValue(1);
6993  }
6994 
6995  CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
6996  } else {
6997  // Collect the legal value parts into potentially illegal values
6998  // that correspond to the original function's return values.
6999  ISD::NodeType AssertOp = ISD::DELETED_NODE;
7000  if (CLI.RetSExt)
7001  AssertOp = ISD::AssertSext;
7002  else if (CLI.RetZExt)
7003  AssertOp = ISD::AssertZext;
7004  unsigned CurReg = 0;
7005  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7006  EVT VT = RetTys[I];
7007  MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7008  unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7009 
7010  ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
7011  NumRegs, RegisterVT, VT, nullptr,
7012  AssertOp));
7013  CurReg += NumRegs;
7014  }
7015 
7016  // For a function returning void, there is no return value. We can't create
7017  // such a node, so we just return a null return value in that case. In
7018  // that case, nothing will actually look at the value.
7019  if (ReturnValues.empty())
7020  return std::make_pair(SDValue(), CLI.Chain);
7021  }
7022 
7023  SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
7024  CLI.DAG.getVTList(RetTys), ReturnValues);
7025  return std::make_pair(Res, CLI.Chain);
7026 }
7027 
7029  SmallVectorImpl<SDValue> &Results,
7030  SelectionDAG &DAG) const {
7031  SDValue Res = LowerOperation(SDValue(N, 0), DAG);
7032  if (Res.getNode())
7033  Results.push_back(Res);
7034 }
7035 
7037  llvm_unreachable("LowerOperation not implemented for this target!");
7038 }
7039 
7040 void
7042  SDValue Op = getNonRegisterValue(V);
7043  assert((Op.getOpcode() != ISD::CopyFromReg ||
7044  cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
7045  "Copy from a reg to the same reg!");
7046  assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
7047 
7048  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7049  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
7050  V->getType());
7051  SDValue Chain = DAG.getEntryNode();
7052 
7053  ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
7055  ? ISD::ANY_EXTEND
7057  RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
7058  PendingExports.push_back(Chain);
7059 }
7060 
7062 
7063 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
7064 /// entry block, return true. This includes arguments used by switches, since
7065 /// the switch may expand into multiple basic blocks.
7066 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
7067  // With FastISel active, we may be splitting blocks, so force creation
7068  // of virtual registers for all non-dead arguments.
7069  if (FastISel)
7070  return A->use_empty();
7071 
7072  const BasicBlock *Entry = A->getParent()->begin();
7073  for (const User *U : A->users())
7074  if (cast<Instruction>(U)->getParent() != Entry || isa<SwitchInst>(U))
7075  return false; // Use not in entry block.
7076 
7077  return true;
7078 }
7079 
7080 void SelectionDAGISel::LowerArguments(const Function &F) {
7081  SelectionDAG &DAG = SDB->DAG;
7082  SDLoc dl = SDB->getCurSDLoc();
7083  const DataLayout &DL = DAG.getDataLayout();
7085 
7086  if (!FuncInfo->CanLowerReturn) {
7087  // Put in an sret pointer parameter before all the other parameters.
7088  SmallVector<EVT, 1> ValueVTs;
7089  ComputeValueVTs(*TLI, DAG.getDataLayout(),
7090  PointerType::getUnqual(F.getReturnType()), ValueVTs);
7091 
7092  // NOTE: Assuming that a pointer will never break down to more than one VT
7093  // or one register.
7095  Flags.setSRet();
7096  MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
7097  ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
7099  Ins.push_back(RetArg);
7100  }
7101 
7102  // Set up the incoming argument description vector.
7103  unsigned Idx = 1;
7104  for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
7105  I != E; ++I, ++Idx) {
7106  SmallVector<EVT, 4> ValueVTs;
7107  ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
7108  bool isArgValueUsed = !I->use_empty();
7109  unsigned PartBase = 0;
7110  Type *FinalType = I->getType();
7112  FinalType = cast<PointerType>(FinalType)->getElementType();
7113  bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
7114  FinalType, F.getCallingConv(), F.isVarArg());
7115  for (unsigned Value = 0, NumValues = ValueVTs.size();
7116  Value != NumValues; ++Value) {
7117  EVT VT = ValueVTs[Value];
7118  Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
7120  unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
7121 
7123  Flags.setZExt();
7125  Flags.setSExt();
7127  Flags.setInReg();
7129  Flags.setSRet();
7131  Flags.setByVal();
7133  Flags.setInAlloca();
7134  // Set the byval flag for CCAssignFn callbacks that don't know about
7135  // inalloca. This way we can know how many bytes we should've allocated
7136  // and how many bytes a callee cleanup function will pop. If we port
7137  // inalloca to more targets, we'll have to add custom inalloca handling
7138  // in the various CC lowering callbacks.
7139  Flags.setByVal();
7140  }
7141  if (Flags.isByVal() || Flags.isInAlloca()) {
7142  PointerType *Ty = cast<PointerType>(I->getType());
7143  Type *ElementTy = Ty->getElementType();
7144  Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
7145  // For ByVal, alignment should be passed from FE. BE will guess if
7146  // this info is not there but there are cases it cannot get right.
7147  unsigned FrameAlign;
7148  if (F.getParamAlignment(Idx))
7149  FrameAlign = F.getParamAlignment(Idx);
7150  else
7151  FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
7152  Flags.setByValAlign(FrameAlign);
7153  }
7155  Flags.setNest();
7156  if (NeedsRegBlock)
7157  Flags.setInConsecutiveRegs();
7158  Flags.setOrigAlign(OriginalAlignment);
7159 
7160  MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
7161  unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT);
7162  for (unsigned i = 0; i != NumRegs; ++i) {
7163  ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
7164  Idx-1, PartBase+i*RegisterVT.getStoreSize());
7165  if (NumRegs > 1 && i == 0)
7166  MyFlags.Flags.setSplit();
7167  // if it isn't first piece, alignment must be 1
7168  else if (i > 0)
7169  MyFlags.Flags.setOrigAlign(1);
7170  Ins.push_back(MyFlags);
7171  }
7172  if (NeedsRegBlock && Value == NumValues - 1)
7173  Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
7174  PartBase += VT.getStoreSize();
7175  }
7176  }
7177 
7178  // Call the target to set up the argument values.
7179  SmallVector<SDValue, 8> InVals;
7180  SDValue NewRoot = TLI->LowerFormalArguments(
7181  DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
7182 
7183  // Verify that the target's LowerFormalArguments behaved as expected.
7184  assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
7185  "LowerFormalArguments didn't return a valid chain!");
7186  assert(InVals.size() == Ins.size() &&
7187  "LowerFormalArguments didn't emit the correct number of values!");
7188  DEBUG({
7189  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
7190  assert(InVals[i].getNode() &&
7191  "LowerFormalArguments emitted a null value!");
7192  assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
7193  "LowerFormalArguments emitted a value with the wrong type!");
7194  }
7195  });
7196 
7197  // Update the DAG with the new chain value resulting from argument lowering.
7198  DAG.setRoot(NewRoot);
7199 
7200  // Set up the argument values.
7201  unsigned i = 0;
7202  Idx = 1;
7203  if (!FuncInfo->CanLowerReturn) {
7204  // Create a virtual register for the sret pointer, and put in a copy
7205  // from the sret argument into it.
7206  SmallVector<EVT, 1> ValueVTs;
7207  ComputeValueVTs(*TLI, DAG.getDataLayout(),
7208  PointerType::getUnqual(F.getReturnType()), ValueVTs);
7209  MVT VT = ValueVTs[0].getSimpleVT();
7210  MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
7211  ISD::NodeType AssertOp = ISD::DELETED_NODE;
7212  SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1,
7213  RegVT, VT, nullptr, AssertOp);
7214 
7216  MachineRegisterInfo& RegInfo = MF.getRegInfo();
7217  unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
7218  FuncInfo->DemoteRegister = SRetReg;
7219  NewRoot =
7220  SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
7221  DAG.setRoot(NewRoot);
7222 
7223  // i indexes lowered arguments. Bump it past the hidden sret argument.
7224  // Idx indexes LLVM arguments. Don't touch it.
7225  ++i;
7226  }
7227 
7228  for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
7229  ++I, ++Idx) {
7230  SmallVector<SDValue, 4> ArgValues;
7231  SmallVector<EVT, 4> ValueVTs;
7232  ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
7233  unsigned NumValues = ValueVTs.size();
7234 
7235  // If this argument is unused then remember its value. It is used to generate
7236  // debugging information.
7237  if (I->use_empty() && NumValues) {
7238  SDB->setUnusedArgValue(I, InVals[i]);
7239 
7240  // Also remember any frame index for use in FastISel.
7241  if (FrameIndexSDNode *FI =
7242  dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
7243  FuncInfo->setArgumentFrameIndex(I, FI->getIndex());
7244  }
7245 
7246  for (unsigned Val = 0; Val != NumValues; ++Val) {
7247  EVT VT = ValueVTs[Val];
7248  MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
7249  unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT);
7250 
7251  if (!I->use_empty()) {
7252  ISD::NodeType AssertOp = ISD::DELETED_NODE;
7254  AssertOp = ISD::AssertSext;
7255  else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt))
7256  AssertOp = ISD::AssertZext;
7257 
7258  ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i],
7259  NumParts, PartVT, VT,
7260  nullptr, AssertOp));
7261  }
7262 
7263  i += NumParts;
7264  }
7265 
7266  // We don't need to do anything else for unused arguments.
7267  if (ArgValues.empty())
7268  continue;
7269 
7270  // Note down frame index.
7271  if (FrameIndexSDNode *FI =
7272  dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
7273  FuncInfo->setArgumentFrameIndex(I, FI->getIndex());
7274 
7275  SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
7276  SDB->getCurSDLoc());
7277 
7278  SDB->setValue(I, Res);
7279  if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
7280  if (LoadSDNode *LNode =
7281  dyn_cast<LoadSDNode>(Res.getOperand(0).getNode()))
7282  if (FrameIndexSDNode *FI =
7283  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
7284  FuncInfo->setArgumentFrameIndex(I, FI->getIndex());
7285  }
7286 
7287  // If this argument is live outside of the entry block, insert a copy from
7288  // wherever we got it to the vreg that other BB's will reference it as.
7289  if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
7290  // If we can, though, try to skip creating an unnecessary vreg.
7291  // FIXME: This isn't very clean... it would be nice to make this more
7292  // general. It's also subtly incompatible with the hacks FastISel
7293  // uses with vregs.
7294  unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
7296  FuncInfo->ValueMap[I] = Reg;
7297  continue;
7298  }
7299  }
7303  }
7304  }
7305 
7306  assert(i == InVals.size() && "Argument register count mismatch!");
7307 
7308  // Finally, if the target has anything special to do, allow it to do so.
7310 }
7311 
7312 /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
7313 /// ensure constants are generated when needed. Remember the virtual registers
7314 /// that need to be added to the Machine PHI nodes as input. We cannot just
7315 /// directly add them, because expansion might result in multiple MBB's for one
7316 /// BB. As such, the start of the BB might correspond to a different MBB than
7317 /// the end.
7318 ///
7319 void
7320 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
7321  const TerminatorInst *TI = LLVMBB->getTerminator();
7322 
7324 
7325  // Check PHI nodes in successors that expect a value to be available from this
7326  // block.
7327  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
7328  const BasicBlock *SuccBB = TI->getSuccessor(succ);
7329  if (!isa<PHINode>(SuccBB->begin())) continue;
7330  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
7331 
7332  // If this terminator has multiple identical successors (common for
7333  // switches), only handle each succ once.
7334  if (!SuccsHandled.insert(SuccMBB).second)
7335  continue;
7336 
7337  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
7338 
7339  // At this point we know that there is a 1-1 correspondence between LLVM PHI
7340  // nodes and Machine PHI nodes, but the incoming operands have not been
7341  // emitted yet.
7342  for (BasicBlock::const_iterator I = SuccBB->begin();
7343  const PHINode *PN = dyn_cast<PHINode>(I); ++I) {
7344  // Ignore dead phi's.
7345  if (PN->use_empty()) continue;
7346 
7347  // Skip empty types
7348  if (PN->getType()->isEmptyTy())
7349  continue;
7350 
7351  unsigned Reg;
7352  const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
7353 
7354  if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
7355  unsigned &RegOut = ConstantsOut[C];
7356  if (RegOut == 0) {
7357  RegOut = FuncInfo.CreateRegs(C->getType());
7358  CopyValueToVirtualRegister(C, RegOut);
7359  }
7360  Reg = RegOut;
7361  } else {
7363  FuncInfo.ValueMap.find(PHIOp);
7364  if (I != FuncInfo.ValueMap.end())
7365  Reg = I->second;
7366  else {
7367  assert(isa<AllocaInst>(PHIOp) &&
7368  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
7369  "Didn't codegen value into a register!??");
7370  Reg = FuncInfo.CreateRegs(PHIOp->getType());
7371  CopyValueToVirtualRegister(PHIOp, Reg);
7372  }
7373  }
7374 
7375  // Remember that this register needs to added to the machine PHI node as
7376  // the input for this MBB.
7377  SmallVector<EVT, 4> ValueVTs;
7378  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7379  ComputeValueVTs(TLI, DAG.getDataLayout(), PN->getType(), ValueVTs);
7380  for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
7381  EVT VT = ValueVTs[vti];
7382  unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
7383  for (unsigned i = 0, e = NumRegisters; i != e; ++i)
7384  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i));
7385  Reg += NumRegisters;
7386  }
7387  }
7388  }
7389 
7390  ConstantsOut.clear();
7391 }
7392 
7393 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
7394 /// is 0.
7396 SelectionDAGBuilder::StackProtectorDescriptor::
7397 AddSuccessorMBB(const BasicBlock *BB,
7398  MachineBasicBlock *ParentMBB,
7399  bool IsLikely,
7400  MachineBasicBlock *SuccMBB) {
7401  // If SuccBB has not been created yet, create it.
7402  if (!SuccMBB) {
7403  MachineFunction *MF = ParentMBB->getParent();
7404  MachineFunction::iterator BBI = ParentMBB;
7405  SuccMBB = MF->CreateMachineBasicBlock(BB);
7406  MF->insert(++BBI, SuccMBB);
7407  }
7408  // Add it as a successor of ParentMBB.
7409  ParentMBB->addSuccessor(
7410  SuccMBB, BranchProbabilityInfo::getBranchWeightStackProtector(IsLikely));
7411  return SuccMBB;
7412 }
7413 
7414 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
7415  MachineFunction::iterator I = MBB;
7416  if (++I == FuncInfo.MF->end())
7417  return nullptr;
7418  return I;
7419 }
7420 
7421 /// During lowering new call nodes can be created (such as memset, etc.).
7422 /// Those will become new roots of the current DAG, but complications arise
7423 /// when they are tail calls. In such cases, the call lowering will update
7424 /// the root, but the builder still needs to know that a tail call has been
7425 /// lowered in order to avoid generating an additional return.
7426 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
7427  // If the node is null, we do have a tail call.
7428  if (MaybeTC.getNode() != nullptr)
7429  DAG.setRoot(MaybeTC);
7430  else
7431  HasTailCall = true;
7432 }
7433 
7434 bool SelectionDAGBuilder::isDense(const CaseClusterVector &Clusters,
7435  unsigned *TotalCases, unsigned First,
7436  unsigned Last) {
7437  assert(Last >= First);
7438  assert(TotalCases[Last] >= TotalCases[First]);
7439 
7440  APInt LowCase = Clusters[First].Low->getValue();
7441  APInt HighCase = Clusters[Last].High->getValue();
7442  assert(LowCase.getBitWidth() == HighCase.getBitWidth());
7443 
7444  // FIXME: A range of consecutive cases has 100% density, but only requires one
7445  // comparison to lower. We should discriminate against such consecutive ranges
7446  // in jump tables.
7447 
7448  uint64_t Diff = (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100);
7449  uint64_t Range = Diff + 1;
7450 
7451  uint64_t NumCases =
7452  TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
7453 
7454  assert(NumCases < UINT64_MAX / 100);
7455  assert(Range >= NumCases);
7456 
7457  return NumCases * 100 >= Range * MinJumpTableDensity;
7458 }
7459 
7460 static inline bool areJTsAllowed(const TargetLowering &TLI) {
7463 }
7464 
7465 bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
7466  unsigned First, unsigned Last,
7467  const SwitchInst *SI,
7468  MachineBasicBlock *DefaultMBB,
7469  CaseCluster &JTCluster) {
7470  assert(First <= Last);
7471 
7472  uint32_t Weight = 0;
7473  unsigned NumCmps = 0;
7474  std::vector<MachineBasicBlock*> Table;
7476  for (unsigned I = First; I <= Last; ++I) {
7477  assert(Clusters[I].Kind == CC_Range);
7478  Weight += Clusters[I].Weight;
7479  assert(Weight >= Clusters[I].Weight && "Weight overflow!");
7480  APInt Low = Clusters[I].Low->getValue();
7481  APInt High = Clusters[I].High->getValue();
7482  NumCmps += (Low == High) ? 1 : 2;
7483  if (I != First) {
7484  // Fill the gap between this and the previous cluster.
7485  APInt PreviousHigh = Clusters[I - 1].High->getValue();
7486  assert(PreviousHigh.slt(Low));
7487  uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
7488  for (uint64_t J = 0; J < Gap; J++)
7489  Table.push_back(DefaultMBB);
7490  }
7491  uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
7492  for (uint64_t J = 0; J < ClusterSize; ++J)
7493  Table.push_back(Clusters[I].MBB);
7494  JTWeights[Clusters[I].MBB] += Clusters[I].Weight;
7495  }
7496 
7497  unsigned NumDests = JTWeights.size();
7498  if (isSuitableForBitTests(NumDests, NumCmps,
7499  Clusters[First].Low->getValue(),
7500  Clusters[Last].High->getValue())) {
7501  // Clusters[First..Last] should be lowered as bit tests instead.
7502  return false;
7503  }
7504 
7505  // Create the MBB that will load from and jump through the table.
7506  // Note: We create it here, but it's not inserted into the function yet.
7507  MachineFunction *CurMF = FuncInfo.MF;
7508  MachineBasicBlock *JumpTableMBB =
7509  CurMF->CreateMachineBasicBlock(SI->getParent());
7510 
7511  // Add successors. Note: use table order for determinism.
7513  for (MachineBasicBlock *Succ : Table) {
7514  if (Done.count(Succ))
7515  continue;
7516  addSuccessorWithWeight(JumpTableMBB, Succ, JTWeights[Succ]);
7517  Done.insert(Succ);
7518  }
7519 
7520  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7521  unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
7522  ->createJumpTableIndex(Table);
7523 
7524  // Set up the jump table info.
7525  JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
7526  JumpTableHeader JTH(Clusters[First].Low->getValue(),
7527  Clusters[Last].High->getValue(), SI->getCondition(),
7528  nullptr, false);
7529  JTCases.emplace_back(std::move(JTH), std::move(JT));
7530 
7531  JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
7532  JTCases.size() - 1, Weight);
7533  return true;
7534 }
7535 
7536 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
7537  const SwitchInst *SI,
7538  MachineBasicBlock *DefaultMBB) {
7539 #ifndef NDEBUG
7540  // Clusters must be non-empty, sorted, and only contain Range clusters.
7541  assert(!Clusters.empty());
7542  for (CaseCluster &C : Clusters)
7543  assert(C.Kind == CC_Range);
7544  for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
7545  assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
7546 #endif
7547 
7548  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7549  if (!areJTsAllowed(TLI))
7550  return;
7551 
7552  const int64_t N = Clusters.size();
7553  const unsigned MinJumpTableSize = TLI.getMinimumJumpTableEntries();
7554 
7555  // TotalCases[i]: Total nbr of cases in Clusters[0..i].
7556  SmallVector<unsigned, 8> TotalCases(N);
7557 
7558  for (unsigned i = 0; i < N; ++i) {
7559  APInt Hi = Clusters[i].High->getValue();
7560  APInt Lo = Clusters[i].Low->getValue();
7561  TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
7562  if (i != 0)
7563  TotalCases[i] += TotalCases[i - 1];
7564  }
7565 
7566  if (N >= MinJumpTableSize && isDense(Clusters, &TotalCases[0], 0, N - 1)) {
7567  // Cheap case: the whole range might be suitable for jump table.
7568  CaseCluster JTCluster;
7569  if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
7570  Clusters[0] = JTCluster;
7571  Clusters.resize(1);
7572  return;
7573  }
7574  }
7575 
7576  // The algorithm below is not suitable for -O0.
7577  if (TM.getOptLevel() == CodeGenOpt::None)
7578  return;
7579 
7580  // Split Clusters into minimum number of dense partitions. The algorithm uses
7581  // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
7582  // for the Case Statement'" (1994), but builds the MinPartitions array in
7583  // reverse order to make it easier to reconstruct the partitions in ascending
7584  // order. In the choice between two optimal partitionings, it picks the one
7585  // which yields more jump tables.
7586 
7587  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
7588  SmallVector<unsigned, 8> MinPartitions(N);
7589  // LastElement[i] is the last element of the partition starting at i.
7590  SmallVector<unsigned, 8> LastElement(N);
7591  // NumTables[i]: nbr of >= MinJumpTableSize partitions from Clusters[i..N-1].
7592  SmallVector<unsigned, 8> NumTables(N);
7593 
7594  // Base case: There is only one way to partition Clusters[N-1].
7595  MinPartitions[N - 1] = 1;
7596  LastElement[N - 1] = N - 1;
7597  assert(MinJumpTableSize > 1);
7598  NumTables[N - 1] = 0;
7599 
7600  // Note: loop indexes are signed to avoid underflow.
7601  for (int64_t i = N - 2; i >= 0; i--) {
7602  // Find optimal partitioning of Clusters[i..N-1].
7603  // Baseline: Put Clusters[i] into a partition on its own.
7604  MinPartitions[i] = MinPartitions[i + 1] + 1;
7605  LastElement[i] = i;
7606  NumTables[i] = NumTables[i + 1];
7607 
7608  // Search for a solution that results in fewer partitions.
7609  for (int64_t j = N - 1; j > i; j--) {
7610  // Try building a partition from Clusters[i..j].
7611  if (isDense(Clusters, &TotalCases[0], i, j)) {
7612  unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
7613  bool IsTable = j - i + 1 >= MinJumpTableSize;
7614  unsigned Tables = IsTable + (j == N - 1 ? 0 : NumTables[j + 1]);
7615 
7616  // If this j leads to fewer partitions, or same number of partitions
7617  // with more lookup tables, it is a better partitioning.
7618  if (NumPartitions < MinPartitions[i] ||
7619  (NumPartitions == MinPartitions[i] && Tables > NumTables[i])) {
7620  MinPartitions[i] = NumPartitions;
7621  LastElement[i] = j;
7622  NumTables[i] = Tables;
7623  }
7624  }
7625  }
7626  }
7627 
7628  // Iterate over the partitions, replacing some with jump tables in-place.
7629  unsigned DstIndex = 0;
7630  for (unsigned First = 0, Last; First < N; First = Last + 1) {
7631  Last = LastElement[First];
7632  assert(Last >= First);
7633  assert(DstIndex <= First);
7634  unsigned NumClusters = Last - First + 1;
7635 
7636  CaseCluster JTCluster;
7637  if (NumClusters >= MinJumpTableSize &&
7638  buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
7639  Clusters[DstIndex++] = JTCluster;
7640  } else {
7641  for (unsigned I = First; I <= Last; ++I)
7642  std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
7643  }
7644  }
7645  Clusters.resize(DstIndex);
7646 }
7647 
7648 bool SelectionDAGBuilder::rangeFitsInWord(const APInt &Low, const APInt &High) {
7649  // FIXME: Using the pointer type doesn't seem ideal.
7650  uint64_t BW = DAG.getDataLayout().getPointerSizeInBits();
7651  uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX - 1) + 1;
7652  return Range <= BW;
7653 }
7654 
7655 bool SelectionDAGBuilder::isSuitableForBitTests(unsigned NumDests,
7656  unsigned NumCmps,
7657  const APInt &Low,
7658  const APInt &High) {
7659  // FIXME: I don't think NumCmps is the correct metric: a single case and a
7660  // range of cases both require only one branch to lower. Just looking at the
7661  // number of clusters and destinations should be enough to decide whether to
7662  // build bit tests.
7663 
7664  // To lower a range with bit tests, the range must fit the bitwidth of a
7665  // machine word.
7666  if (!rangeFitsInWord(Low, High))
7667  return false;
7668 
7669  // Decide whether it's profitable to lower this range with bit tests. Each
7670  // destination requires a bit test and branch, and there is an overall range
7671  // check branch. For a small number of clusters, separate comparisons might be
7672  // cheaper, and for many destinations, splitting the range might be better.
7673  return (NumDests == 1 && NumCmps >= 3) ||
7674  (NumDests == 2 && NumCmps >= 5) ||
7675  (NumDests == 3 && NumCmps >= 6);
7676 }
7677 
7678 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
7679  unsigned First, unsigned Last,
7680  const SwitchInst *SI,
7681  CaseCluster &BTCluster) {
7682  assert(First <= Last);
7683  if (First == Last)
7684  return false;
7685 
7686  BitVector Dests(FuncInfo.MF->getNumBlockIDs());
7687  unsigned NumCmps = 0;
7688  for (int64_t I = First; I <= Last; ++I) {
7689  assert(Clusters[I].Kind == CC_Range);
7690  Dests.set(Clusters[I].MBB->getNumber());
7691  NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
7692  }
7693  unsigned NumDests = Dests.count();
7694 
7695  APInt Low = Clusters[First].Low->getValue();
7696  APInt High = Clusters[Last].High->getValue();
7697  assert(Low.slt(High));
7698 
7699  if (!isSuitableForBitTests(NumDests, NumCmps, Low, High))
7700  return false;
7701 
7702  APInt LowBound;
7703  APInt CmpRange;
7704 
7705  const int BitWidth = DAG.getTargetLoweringInfo()
7706  .getPointerTy(DAG.getDataLayout())
7707  .getSizeInBits();
7708  assert(rangeFitsInWord(Low, High) && "Case range must fit in bit mask!");
7709 
7710  if (Low.isNonNegative() && High.slt(BitWidth)) {
7711  // Optimize the case where all the case values fit in a
7712  // word without having to subtract minValue. In this case,
7713  // we can optimize away the subtraction.
7714  LowBound = APInt::getNullValue(Low.getBitWidth());
7715  CmpRange = High;
7716  } else {
7717  LowBound = Low;
7718  CmpRange = High - Low;
7719  }
7720 
7721  CaseBitsVector CBV;
7722  uint32_t TotalWeight = 0;
7723  for (unsigned i = First; i <= Last; ++i) {
7724  // Find the CaseBits for this destination.
7725  unsigned j;
7726  for (j = 0; j < CBV.size(); ++j)
7727  if (CBV[j].BB == Clusters[i].MBB)
7728  break;
7729  if (j == CBV.size())
7730  CBV.push_back(CaseBits(0, Clusters[i].MBB, 0, 0));
7731  CaseBits *CB = &CBV[j];
7732 
7733  // Update Mask, Bits and ExtraWeight.
7734  uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
7735  uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
7736  assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
7737  CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
7738  CB->Bits += Hi - Lo + 1;
7739  CB->ExtraWeight += Clusters[i].Weight;
7740  TotalWeight += Clusters[i].Weight;
7741  assert(TotalWeight >= Clusters[i].Weight && "Weight overflow!");
7742  }
7743 
7744  BitTestInfo BTI;
7745  std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
7746  // Sort by weight first, number of bits second.
7747  if (a.ExtraWeight != b.ExtraWeight)
7748  return a.ExtraWeight > b.ExtraWeight;
7749  return a.Bits > b.Bits;
7750  });
7751 
7752  for (auto &CB : CBV) {
7753  MachineBasicBlock *BitTestBB =
7755  BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraWeight));
7756  }
7757  BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
7758  SI->getCondition(), -1U, MVT::Other, false, nullptr,
7759  nullptr, std::move(BTI));
7760 
7761  BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
7762  BitTestCases.size() - 1, TotalWeight);
7763  return true;
7764 }
7765 
7766 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
7767  const SwitchInst *SI) {
7768 // Partition Clusters into as few subsets as possible, where each subset has a
7769 // range that fits in a machine word and has <= 3 unique destinations.
7770 
7771 #ifndef NDEBUG
7772  // Clusters must be sorted and contain Range or JumpTable clusters.
7773  assert(!Clusters.empty());
7774  assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
7775  for (const CaseCluster &C : Clusters)
7776  assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
7777  for (unsigned i = 1; i < Clusters.size(); ++i)
7778  assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
7779 #endif
7780 
7781  // The algorithm below is not suitable for -O0.
7782  if (TM.getOptLevel() == CodeGenOpt::None)
7783  return;
7784 
7785  // If target does not have legal shift left, do not emit bit tests at all.
7786  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7787  EVT PTy = TLI.getPointerTy(DAG.getDataLayout());
7788  if (!TLI.isOperationLegal(ISD::SHL, PTy))
7789  return;
7790 
7791  int BitWidth = PTy.getSizeInBits();
7792  const int64_t N = Clusters.size();
7793 
7794  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
7795  SmallVector<unsigned, 8> MinPartitions(N);
7796  // LastElement[i] is the last element of the partition starting at i.
7797  SmallVector<unsigned, 8> LastElement(N);
7798 
7799  // FIXME: This might not be the best algorithm for finding bit test clusters.
7800 
7801  // Base case: There is only one way to partition Clusters[N-1].
7802  MinPartitions[N - 1] = 1;
7803  LastElement[N - 1] = N - 1;
7804 
7805  // Note: loop indexes are signed to avoid underflow.
7806  for (int64_t i = N - 2; i >= 0; --i) {
7807  // Find optimal partitioning of Clusters[i..N-1].
7808  // Baseline: Put Clusters[i] into a partition on its own.
7809  MinPartitions[i] = MinPartitions[i + 1] + 1;
7810  LastElement[i] = i;
7811 
7812  // Search for a solution that results in fewer partitions.
7813  // Note: the search is limited by BitWidth, reducing time complexity.
7814  for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
7815  // Try building a partition from Clusters[i..j].
7816 
7817  // Check the range.
7818  if (!rangeFitsInWord(Clusters[i].Low->getValue(),
7819  Clusters[j].High->getValue()))
7820  continue;
7821 
7822  // Check nbr of destinations and cluster types.
7823  // FIXME: This works, but doesn't seem very efficient.
7824  bool RangesOnly = true;
7825  BitVector Dests(FuncInfo.MF->getNumBlockIDs());
7826  for (int64_t k = i; k <= j; k++) {
7827  if (Clusters[k].Kind != CC_Range) {
7828  RangesOnly = false;
7829  break;
7830  }
7831  Dests.set(Clusters[k].MBB->getNumber());
7832  }
7833  if (!RangesOnly || Dests.count() > 3)
7834  break;
7835 
7836  // Check if it's a better partition.
7837  unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
7838  if (NumPartitions < MinPartitions[i]) {
7839  // Found a better partition.
7840  MinPartitions[i] = NumPartitions;
7841  LastElement[i] = j;
7842  }
7843  }
7844  }
7845 
7846  // Iterate over the partitions, replacing with bit-test clusters in-place.
7847  unsigned DstIndex = 0;
7848  for (unsigned First = 0, Last; First < N; First = Last + 1) {
7849  Last = LastElement[First];
7850  assert(First <= Last);
7851  assert(DstIndex <= First);
7852 
7853  CaseCluster BitTestCluster;
7854  if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
7855  Clusters[DstIndex++] = BitTestCluster;
7856  } else {
7857  size_t NumClusters = Last - First + 1;
7858  std::memmove(&Clusters[DstIndex], &Clusters[First],
7859  sizeof(Clusters[0]) * NumClusters);
7860  DstIndex += NumClusters;
7861  }
7862  }
7863  Clusters.resize(DstIndex);
7864 }
7865 
7866 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
7867  MachineBasicBlock *SwitchMBB,
7868  MachineBasicBlock *DefaultMBB) {
7869  MachineFunction *CurMF = FuncInfo.MF;
7870  MachineBasicBlock *NextMBB = nullptr;
7871  MachineFunction::iterator BBI = W.MBB;
7872  if (++BBI != FuncInfo.MF->end())
7873  NextMBB = BBI;
7874 
7875  unsigned Size = W.LastCluster - W.FirstCluster + 1;
7876 
7878 
7879  if (Size == 2 && W.MBB == SwitchMBB) {
7880  // If any two of the cases has the same destination, and if one value
7881  // is the same as the other, but has one bit unset that the other has set,
7882  // use bit manipulation to do two compares at once. For example:
7883  // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
7884  // TODO: This could be extended to merge any 2 cases in switches with 3
7885  // cases.
7886  // TODO: Handle cases where W.CaseBB != SwitchBB.
7887  CaseCluster &Small = *W.FirstCluster;
7888  CaseCluster &Big = *W.LastCluster;
7889 
7890  if (Small.Low == Small.High && Big.Low == Big.High &&
7891  Small.MBB == Big.MBB) {
7892  const APInt &SmallValue = Small.Low->getValue();
7893  const APInt &BigValue = Big.Low->getValue();
7894 
7895  // Check that there is only one bit different.
7896  APInt CommonBit = BigValue ^ SmallValue;
7897  if (CommonBit.isPowerOf2()) {
7898  SDValue CondLHS = getValue(Cond);
7899  EVT VT = CondLHS.getValueType();
7900  SDLoc DL = getCurSDLoc();
7901 
7902  SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
7903  DAG.getConstant(CommonBit, DL, VT));
7904  SDValue Cond = DAG.getSetCC(
7905  DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
7906  ISD::SETEQ);
7907 
7908  // Update successor info.
7909  // Both Small and Big will jump to Small.BB, so we sum up the weights.
7910  addSuccessorWithWeight(SwitchMBB, Small.MBB, Small.Weight + Big.Weight);
7911  addSuccessorWithWeight(
7912  SwitchMBB, DefaultMBB,
7913  // The default destination is the first successor in IR.
7914  BPI ? BPI->getEdgeWeight(SwitchMBB->getBasicBlock(), (unsigned)0)
7915  : 0);
7916 
7917  // Insert the true branch.
7918  SDValue BrCond =
7919  DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
7920  DAG.getBasicBlock(Small.MBB));
7921  // Insert the false branch.
7922  BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
7923  DAG.getBasicBlock(DefaultMBB));
7924 
7925  DAG.setRoot(BrCond);
7926  return;
7927  }
7928  }
7929  }
7930 
7931  if (TM.getOptLevel() != CodeGenOpt::None) {
7932  // Order cases by weight so the most likely case will be checked first.
7933  std::sort(W.FirstCluster, W.LastCluster + 1,
7934  [](const CaseCluster &a, const CaseCluster &b) {
7935  return a.Weight > b.Weight;
7936  });
7937 
7938  // Rearrange the case blocks so that the last one falls through if possible
7939  // without without changing the order of weights.
7940  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
7941  --I;
7942  if (I->Weight > W.LastCluster->Weight)
7943  break;
7944  if (I->Kind == CC_Range && I->MBB == NextMBB) {
7945  std::swap(*I, *W.LastCluster);
7946  break;
7947  }
7948  }
7949  }
7950 
7951  // Compute total weight.
7952  uint32_t UnhandledWeights = 0;
7953  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) {
7954  UnhandledWeights += I->Weight;
7955  assert(UnhandledWeights >= I->Weight && "Weight overflow!");
7956  }
7957 
7958  MachineBasicBlock *CurMBB = W.MBB;
7959  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
7960  MachineBasicBlock *Fallthrough;
7961  if (I == W.LastCluster) {
7962  // For the last cluster, fall through to the default destination.
7963  Fallthrough = DefaultMBB;
7964  } else {
7965  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
7966  CurMF->insert(BBI, Fallthrough);
7967  // Put Cond in a virtual register to make it available from the new blocks.
7968  ExportFromCurrentBlock(Cond);
7969  }
7970 
7971  switch (I->Kind) {
7972  case CC_JumpTable: {
7973  // FIXME: Optimize away range check based on pivot comparisons.
7974  JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
7975  JumpTable *JT = &JTCases[I->JTCasesIndex].second;
7976 
7977  // The jump block hasn't been inserted yet; insert it here.
7978  MachineBasicBlock *JumpMBB = JT->MBB;
7979  CurMF->insert(BBI, JumpMBB);
7980  addSuccessorWithWeight(CurMBB, Fallthrough);
7981  addSuccessorWithWeight(CurMBB, JumpMBB);
7982 
7983  // The jump table header will be inserted in our current block, do the
7984  // range check, and fall through to our fallthrough block.
7985  JTH->HeaderBB = CurMBB;
7986  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
7987 
7988  // If we're in the right place, emit the jump table header right now.
7989  if (CurMBB == SwitchMBB) {
7990  visitJumpTableHeader(*JT, *JTH, SwitchMBB);
7991  JTH->Emitted = true;
7992  }
7993  break;
7994  }
7995  case CC_BitTests: {
7996  // FIXME: Optimize away range check based on pivot comparisons.
7997  BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
7998 
7999  // The bit test blocks haven't been inserted yet; insert them here.
8000  for (BitTestCase &BTC : BTB->Cases)
8001  CurMF->insert(BBI, BTC.ThisBB);
8002 
8003  // Fill in fields of the BitTestBlock.
8004  BTB->Parent = CurMBB;
8005  BTB->Default = Fallthrough;
8006 
8007  // If we're in the right place, emit the bit test header header right now.
8008  if (CurMBB ==SwitchMBB) {
8009  visitBitTestHeader(*BTB, SwitchMBB);
8010  BTB->Emitted = true;
8011  }
8012  break;
8013  }
8014  case CC_Range: {
8015  const Value *RHS, *LHS, *MHS;
8016  ISD::CondCode CC;
8017  if (I->Low == I->High) {
8018  // Check Cond == I->Low.
8019  CC = ISD::SETEQ;
8020  LHS = Cond;
8021  RHS=I->Low;
8022  MHS = nullptr;
8023  } else {
8024  // Check I->Low <= Cond <= I->High.
8025  CC = ISD::SETLE;
8026  LHS = I->Low;
8027  MHS = Cond;
8028  RHS = I->High;
8029  }
8030 
8031  // The false weight is the sum of all unhandled cases.
8032  UnhandledWeights -= I->Weight;
8033  CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Weight,
8034  UnhandledWeights);
8035 
8036  if (CurMBB == SwitchMBB)
8037  visitSwitchCase(CB, SwitchMBB);
8038  else
8039  SwitchCases.push_back(CB);
8040 
8041  break;
8042  }
8043  }
8044  CurMBB = Fallthrough;
8045  }
8046 }
8047 
8048 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
8049  CaseClusterIt First,
8050  CaseClusterIt Last) {
8051  return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
8052  if (X.Weight != CC.Weight)
8053  return X.Weight > CC.Weight;
8054 
8055  // Ties are broken by comparing the case value.
8056  return X.Low->getValue().slt(CC.Low->getValue());
8057  });
8058 }
8059 
8060 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
8061  const SwitchWorkListItem &W,
8062  Value *Cond,
8063  MachineBasicBlock *SwitchMBB) {
8064  assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
8065  "Clusters not sorted?");
8066 
8067  assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
8068 
8069  // Balance the tree based on branch weights to create a near-optimal (in terms
8070  // of search time given key frequency) binary search tree. See e.g. Kurt
8071  // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
8072  CaseClusterIt LastLeft = W.FirstCluster;
8073  CaseClusterIt FirstRight = W.LastCluster;
8074  uint32_t LeftWeight = LastLeft->Weight;
8075  uint32_t RightWeight = FirstRight->Weight;
8076 
8077  // Move LastLeft and FirstRight towards each other from opposite directions to
8078  // find a partitioning of the clusters which balances the weight on both
8079  // sides. If LeftWeight and RightWeight are equal, alternate which side is
8080  // taken to ensure 0-weight nodes are distributed evenly.
8081  unsigned I = 0;
8082  while (LastLeft + 1 < FirstRight) {
8083  if (LeftWeight < RightWeight || (LeftWeight == RightWeight && (I & 1)))
8084  LeftWeight += (++LastLeft)->Weight;
8085  else
8086  RightWeight += (--FirstRight)->Weight;
8087  I++;
8088  }
8089 
8090  for (;;) {
8091  // Our binary search tree differs from a typical BST in that ours can have up
8092  // to three values in each leaf. The pivot selection above doesn't take that
8093  // into account, which means the tree might require more nodes and be less
8094  // efficient. We compensate for this here.
8095 
8096  unsigned NumLeft = LastLeft - W.FirstCluster + 1;
8097  unsigned NumRight = W.LastCluster - FirstRight + 1;
8098 
8099  if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
8100  // If one side has less than 3 clusters, and the other has more than 3,
8101  // consider taking a cluster from the other side.
8102 
8103  if (NumLeft < NumRight) {
8104  // Consider moving the first cluster on the right to the left side.
8105  CaseCluster &CC = *FirstRight;
8106  unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
8107  unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
8108  if (LeftSideRank <= RightSideRank) {
8109  // Moving the cluster to the left does not demote it.
8110  ++LastLeft;
8111  ++FirstRight;
8112  continue;
8113  }
8114  } else {
8115  assert(NumRight < NumLeft);
8116  // Consider moving the last element on the left to the right side.
8117  CaseCluster &CC = *LastLeft;
8118  unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
8119  unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
8120  if (RightSideRank <= LeftSideRank) {
8121  // Moving the cluster to the right does not demot it.
8122  --LastLeft;
8123  --FirstRight;
8124  continue;
8125  }
8126  }
8127  }
8128  break;
8129  }
8130 
8131  assert(LastLeft + 1 == FirstRight);
8132  assert(LastLeft >= W.FirstCluster);
8133  assert(FirstRight <= W.LastCluster);
8134 
8135  // Use the first element on the right as pivot since we will make less-than
8136  // comparisons against it.
8137  CaseClusterIt PivotCluster = FirstRight;
8138  assert(PivotCluster > W.FirstCluster);
8139  assert(PivotCluster <= W.LastCluster);
8140 
8141  CaseClusterIt FirstLeft = W.FirstCluster;
8142  CaseClusterIt LastRight = W.LastCluster;
8143 
8144  const ConstantInt *Pivot = PivotCluster->Low;
8145 
8146  // New blocks will be inserted immediately after the current one.
8147  MachineFunction::iterator BBI = W.MBB;
8148  ++BBI;
8149 
8150  // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
8151  // we can branch to its destination directly if it's squeezed exactly in
8152  // between the known lower bound and Pivot - 1.
8153  MachineBasicBlock *LeftMBB;
8154  if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
8155  FirstLeft->Low == W.GE &&
8156  (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
8157  LeftMBB = FirstLeft->MBB;
8158  } else {
8159  LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
8160  FuncInfo.MF->insert(BBI, LeftMBB);
8161  WorkList.push_back({LeftMBB, FirstLeft, LastLeft, W.GE, Pivot});
8162  // Put Cond in a virtual register to make it available from the new blocks.
8163  ExportFromCurrentBlock(Cond);
8164  }
8165 
8166  // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
8167  // single cluster, RHS.Low == Pivot, and we can branch to its destination
8168  // directly if RHS.High equals the current upper bound.
8169  MachineBasicBlock *RightMBB;
8170  if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
8171  W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
8172  RightMBB = FirstRight->MBB;
8173  } else {
8174  RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
8175  FuncInfo.MF->insert(BBI, RightMBB);
8176  WorkList.push_back({RightMBB, FirstRight, LastRight, Pivot, W.LT});
8177  // Put Cond in a virtual register to make it available from the new blocks.
8178  ExportFromCurrentBlock(Cond);
8179  }
8180 
8181  // Create the CaseBlock record that will be used to lower the branch.
8182  CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
8183  LeftWeight, RightWeight);
8184 
8185  if (W.MBB == SwitchMBB)
8186  visitSwitchCase(CB, SwitchMBB);
8187  else
8188  SwitchCases.push_back(CB);
8189 }
8190 
8191 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
8192  // Extract cases from the switch.
8194  CaseClusterVector Clusters;
8195  Clusters.reserve(SI.getNumCases());
8196  for (auto I : SI.cases()) {
8197  MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
8198  const ConstantInt *CaseVal = I.getCaseValue();
8199  uint32_t Weight =
8200  BPI ? BPI->getEdgeWeight(SI.getParent(), I.getSuccessorIndex()) : 0;
8201  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Weight));
8202  }
8203 
8204  MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
8205 
8206  // Cluster adjacent cases with the same destination. We do this at all
8207  // optimization levels because it's cheap to do and will make codegen faster
8208  // if there are many clusters.
8209  sortAndRangeify(Clusters);
8210 
8211  if (TM.getOptLevel() != CodeGenOpt::None) {
8212  // Replace an unreachable default with the most popular destination.
8213  // FIXME: Exploit unreachable default more aggressively.
8214  bool UnreachableDefault =
8215  isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
8216  if (UnreachableDefault && !Clusters.empty()) {
8218  unsigned MaxPop = 0;
8219  const BasicBlock *MaxBB = nullptr;
8220  for (auto I : SI.cases()) {
8221  const BasicBlock *BB = I.getCaseSuccessor();
8222  if (++Popularity[BB] > MaxPop) {
8223  MaxPop = Popularity[BB];
8224  MaxBB = BB;
8225  }
8226  }
8227  // Set new default.
8228  assert(MaxPop > 0 && MaxBB);
8229  DefaultMBB = FuncInfo.MBBMap[MaxBB];
8230 
8231  // Remove cases that were pointing to the destination that is now the
8232  // default.
8233  CaseClusterVector New;
8234  New.reserve(Clusters.size());
8235  for (CaseCluster &CC : Clusters) {
8236  if (CC.MBB != DefaultMBB)
8237  New.push_back(CC);
8238  }
8239  Clusters = std::move(New);
8240  }
8241  }
8242 
8243  // If there is only the default destination, jump there directly.
8244  MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
8245  if (Clusters.empty()) {
8246  SwitchMBB->addSuccessor(DefaultMBB);
8247  if (DefaultMBB != NextBlock(SwitchMBB)) {
8248  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
8249  getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
8250  }
8251  return;
8252  }
8253 
8254  findJumpTables(Clusters, &SI, DefaultMBB);
8255  findBitTestClusters(Clusters, &SI);
8256 
8257  DEBUG({
8258  dbgs() << "Case clusters: ";
8259  for (const CaseCluster &C : Clusters) {
8260  if (C.Kind == CC_JumpTable) dbgs() << "JT:";
8261  if (C.Kind == CC_BitTests) dbgs() << "BT:";
8262 
8263  C.Low->getValue().print(dbgs(), true);
8264  if (C.Low != C.High) {
8265  dbgs() << '-';
8266  C.High->getValue().print(dbgs(), true);
8267  }
8268  dbgs() << ' ';
8269  }
8270  dbgs() << '\n';
8271  });
8272 
8273  assert(!Clusters.empty());
8274  SwitchWorkList WorkList;
8275  CaseClusterIt First = Clusters.begin();
8276  CaseClusterIt Last = Clusters.end() - 1;
8277  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr});
8278 
8279  while (!WorkList.empty()) {
8280  SwitchWorkListItem W = WorkList.back();
8281  WorkList.pop_back();
8282  unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
8283 
8284  if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None) {
8285  // For optimized builds, lower large range as a balanced binary tree.
8286  splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
8287  continue;
8288  }
8289 
8290  lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
8291  }
8292 }
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
void setHasStackMap(bool s=true)
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:641
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, SDLoc, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array...
static unsigned getTruncatedArgReg(const SDValue &N)
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
void setAllowReciprocal(bool b)
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:450
BasicBlock * getSuccessor(unsigned i) const
ReturnInst - Return a value (possibly void), from a function.
Value * getValueOperand()
Definition: Instructions.h:406
const Value * getCalledValue() const
getCalledValue - Get a pointer to the function that is invoked by this instruction.
std::vector< BitTestBlock > BitTestCases
BitTestCases - Vector of BitTestBlock structures used to communicate SwitchInst code generation infor...
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
bool describes(const Function *F) const
Check if this subprogram decribes the given function.
static MVT getIntegerVT(unsigned BitWidth)
void setByValAlign(unsigned A)
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:481
The memory access reads data.
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
SelectionDAGBuilder * SDB
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
AtomicOrdering getFailureOrdering() const
Returns the ordering constraint on this cmpxchg.
Definition: Instructions.h:597
AsmDialect getDialect() const
Definition: InlineAsm.h:70
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:537
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, SDLoc DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
EmitTargetCodeForStrcpy - Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libcall.
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, unsigned FixedArgs=-1)
SDValue getValue(unsigned R) const
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:679
The memory access writes data.
bool findValue(const Value *V) const
unsigned getTypeIDFor(const GlobalValue *TI)
getTypeIDFor - Return the type id for the specified typeinfo.
ExtractValueInst - This instruction extracts a struct member or array element value from an aggregate...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:175
*p = old <signed v ? old : v
Definition: Instructions.h:704
iterator_range< CaseIt > cases()
cases - iteration adapter for range-for loops.
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
LLVM Argument representation.
Definition: Argument.h:35
SDValue getCopyToReg(SDValue Chain, SDLoc dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:522
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, SDLoc DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd)...
Definition: SelectionDAG.h:646
static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse)
Scale down both weights to fit into uint32_t.
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:111
bool hasName() const
Definition: Value.h:228
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:42
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:292
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
bool isVolatile() const
isVolatile - Return true if this is a store to a volatile memory location.
Definition: Instructions.h:351
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
SynchronizationScope getSynchScope() const
Definition: Instructions.h:383
const TargetLibraryInfo * LibInfo
ArrayRef< unsigned > getIndices() const
Sign extended before/after call.
Definition: Attributes.h:105
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static MVT getVectorVT(MVT VT, unsigned NumElements)
ValTy * getArgument(unsigned ArgNo) const
Definition: CallSite.h:119
void LowerStatepoint(ImmutableStatepoint Statepoint, MachineBasicBlock *LandingPad=nullptr)
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:39
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Force argument to be passed in register.
Definition: Attributes.h:78
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
AtomicOrdering getSuccessOrdering() const
Returns the ordering constraint on this cmpxchg.
Definition: Instructions.h:592
FenceInst - an instruction for ordering other memory operations.
Definition: Instructions.h:445
InstrTy * getInstruction() const
Definition: CallSite.h:82
SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
void CopyValueToVirtualRegister(const Value *V, unsigned Reg)
AtomicCmpXchgInst - an instruction that atomically checks whether a specified value is in a memory lo...
Definition: Instructions.h:515
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static bool getUniformBase(Value *&Ptr, SDValue &Base, SDValue &Index, SelectionDAGBuilder *SDB)
CallLoweringInfo & setDebugLoc(SDLoc dl)
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6...
DILocalScope * getScope() const
Get the local scope for this variable.
SDValue getMergeValues(ArrayRef< SDValue > Ops, SDLoc dl)
Create a MERGE_VALUES node from the given operands.
unsigned getNumOperands() const
Definition: User.h:138
Nested function static chain.
Definition: Attributes.h:82
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
Definition: Statepoint.h:283
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:285
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, SDLoc DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:684
MCSymbol * getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen...
Definition: MCContext.cpp:146
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:324
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
A Stackmap instruction captures the location of live variables at its position in the instruction str...
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:286
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:585
CallInst - This class represents a function call, abstracting a target machine's calling convention...
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
bool usesUnderscoreSetJmp() const
Determine if we should use _setjmp or setjmp to implement llvm.setjmp.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:276
const std::string & getAsmString() const
Definition: InlineAsm.h:82
SDValue getBasicBlock(MachineBasicBlock *MBB)
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:228
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
*p = old <unsigned v ? old : v
Definition: Instructions.h:708
const TargetSelectionDAGInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:288
void addLiveIn(unsigned Reg)
Adds the specified register as a live in.
void setValue(const Value *V, SDValue NewN)
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:378
unsigned getID() const
getID() - Return the register class ID number.
void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
*p = old >unsigned v ? old : v
Definition: Instructions.h:706
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:181
unsigned getSizeInBits() const
ShuffleVectorInst - This instruction constructs a fixed permutation of two input vectors.
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const
For some targets, an LLVM struct type must be broken down into multiple simple types, but the calling convention specifies that the entire struct must be passed in a block of consecutive registers.
static cl::opt< bool > EnableFMFInDAG("enable-fmf-dag", cl::init(false), cl::Hidden, cl::desc("Enable fast-math-flags for DAG nodes"))
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:679
Type * getReturnType() const
Definition: Function.cpp:233
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
arg_iterator arg_end()
Definition: Function.h:480
virtual bool isZExtFree(Type *, Type *) const
Return true if any actual instruction that defines a value of type Ty1 implicitly zero-extends the va...
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:740
const SDValue & getOperand(unsigned Num) const
F(f)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class represents a sign extension of integer types.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
LoadInst - an instruction for reading from memory.
Definition: Instructions.h:177
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:318
AtomicRMWInst - an instruction that atomically reads a memory location, combines it with another valu...
Definition: Instructions.h:674
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
CvtCode
CvtCode enum - This enum defines the various converts CONVERT_RNDSAT supports.
Definition: ISDOpcodes.h:885
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: Type.cpp:216
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:344
Same for subtraction.
Definition: ISDOpcodes.h:231
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:238
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
Definition: Attributes.cpp:956
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
void setNoSignedZeros(bool b)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
*p = old >signed v ? old : v
Definition: Instructions.h:702
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, unsigned f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
getMachineMemOperand - Allocate a new MachineMemOperand.
DebugLoc getCurDebugLoc() const
uint64_t getOffset() const
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:658
op_iterator op_begin()
Definition: User.h:183
void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter)
Add a dbg_value SDNode.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, unsigned Opc, uint32_t TW, uint32_t FW)
FindMergedConditions - If Cond is an expression like.
SDValue getExternalSymbol(const char *Sym, EVT VT)
Constant * getMask() const
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:172
SDValue getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
static MachinePointerInfo getFixedStack(int FI, int64_t offset=0)
getFixedStack - Return a MachinePointerInfo record that refers to the the specified FrameIndex...
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:150
virtual bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
pointsToConstantMemory - If the specified memory location is known to be constant, return true.
void setCurrentCallSite(unsigned Site)
setCurrentCallSite - Set the call site currently being processed.
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
MachineJumpTableInfo * getOrCreateJumpTableInfo(unsigned JTEntryKind)
getOrCreateJumpTableInfo - Get the JumpTableInfo for this function, if it does already exist...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
bool isSingleValueType() const
isSingleValueType - Return true if the type is a valid type for a register in codegen.
Definition: Type.h:250
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic...
Definition: ISDOpcodes.h:109
SelectPatternFlavor matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
StackProtectorDescriptor SPDescriptor
A StackProtectorDescriptor structure used to communicate stack protector information in between Selec...
ArrayRef< unsigned > getIndices() const
Instruction * getFirstNonPHIOrDbg()
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:172
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
void setNoSignedWrap(bool b)
lazy value info
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
IterTy arg_end() const
Definition: CallSite.h:157
BlockAddress - The address of a basic block.
Definition: Constants.h:802
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
bool isUnconditional() const
MachineMemOperand - A description of a memory reference used in the backend.
unsigned DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
void setHasPatchPoint(bool s=true)
static Value * getPointerOperand(Instruction &Inst)
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:289
SDValue getMemset(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo)
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:618
bool noSignedZeros() const
Definition: Operator.h:191
void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:79
const HexagonInstrInfo * TII
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, SDLoc dl)
GetExponent - Get the exponent:
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:106
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:308
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false)
Shift and rotation operations.
Definition: ISDOpcodes.h:332
globalsmodref aa
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:407
StructType - Class to represent struct types.
Definition: DerivedTypes.h:191
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
bool hasOptimizedCodeGen(LibFunc::Func F) const
Tests if the function is both available and a candidate for optimized code generation.
bool isDereferenceablePointer(const Value *V, const DataLayout &DL, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
isDereferenceablePointer - Return true if this is always a dereferenceable pointer.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:283
void visitJumpTable(JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, unsigned NumOps, bool isSigned, SDLoc dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
CallLoweringInfo & setChain(SDValue InChain)
Constant * ConstantFoldLoadFromConstPtr(Constant *C, const DataLayout &DL)
ConstantFoldLoadFromConstPtr - Return the value that a load from C would produce if it is constant an...
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:181
bool isReg() const
isReg - Tests if this is a MO_Register operand.
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, SDLoc dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr) const
getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from this value and returns the resu...
SDValue getTargetGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:467
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const TargetRegisterClass * getRegClass(unsigned Reg) const
getRegClass - Return the register class of the specified virtual register.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:458
const Value * getCalledValue() const
getCalledValue - Get a pointer to the function that is invoked by this instruction ...
SmallVector< EVT, 4 > ValueVTs
ValueVTs - The value types of the values, which may not be legal, and may need be promoted or synthes...
iterator begin() const
begin/end - Return all of the registers in this class.
Reg
All possible values of the reg field in the ModR/M byte.
SDDbgValue * getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool IsIndirect, uint64_t Off, DebugLoc DL, unsigned O)
Creates a SDDbgValue node.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
bool isSized(SmallPtrSetImpl< const Type * > *Visited=nullptr) const
isSized - Return true if it makes sense to take the size of this type.
Definition: Type.h:268
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:210
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:687
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
SynchronizationScope
Definition: Instructions.h:49
static bool isRegDefEarlyClobberKind(unsigned Flag)
Definition: InlineAsm.h:318
static unsigned convertMemFlagWordToMatchingFlagWord(unsigned InputFlag)
Definition: InlineAsm.h:306
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
Definition: ValueTypes.h:183
static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandExp - Lower an exp intrinsic.
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, SDLoc DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
void setByValSize(unsigned S)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
The memory access is volatile.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
void setFunctionContextIndex(int I)
static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog10 - Lower a log10 intrinsic.
const MachineInstrBuilder & addImm(int64_t Val) const
addImm - Add a new immediate operand.
Unsigned from Unsigned.
Definition: ISDOpcodes.h:895
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Hidden pointer to structure to return.
Definition: Attributes.h:114
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:571
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned getEVTAlignment(EVT MemoryVT) const
getEVTAlignment - Compute the default alignment value for the given type.
SynchronizationScope getSynchScope() const
Definition: Instructions.h:261
SmallVector< ISD::InputArg, 32 > Ins
AtomicOrdering
Definition: Instructions.h:38
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:581
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:91
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:117
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
bool isStatepoint(const ImmutableCallSite &CS)
Definition: Statepoint.cpp:22
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:442
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first...
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
SDValue getCALLSEQ_START(SDValue Chain, SDValue Op, SDLoc DL)
Return a new CALLSEQ_START node, which always must have a glue result (to ensure it's not CSE'd)...
Definition: SelectionDAG.h:637
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:30
FunctionType - Class to represent function types.
Definition: DerivedTypes.h:96
ConstantExpr - a constant value that is initialized with an expression using other constant values...
Definition: Constants.h:852
AtomicOrdering getOrdering() const
Returns the ordering constraint on this RMW.
Definition: Instructions.h:769
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:393
bool usesUnderscoreLongJmp() const
Determine if we should use _longjmp or longjmp to implement llvm.longjmp.
LLVMContext & getContext() const
getContext - Return the LLVMContext in which this type was uniqued.
Definition: Type.h:125
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
const LiveOutInfo * GetLiveOutRegInfo(unsigned Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
Definition: CallSite.h:91
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
setCallSiteBeginLabel - Map the begin label for a call site.
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:557
void GetReturnInfo(Type *ReturnType, AttributeSet attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:107
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:473
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:351
virtual EVT getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer argument or retu...
This contains information for each constraint that we are lowering.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo.ValueMap for a virtual register.
This instruction compares its operands according to the predicate given to the constructor.
BasicBlock * getSuccessor(unsigned i) const
SmallVector< ISD::OutputArg, 32 > Outs
This class represents a no-op cast from one type to another.
unsigned getCurrentCallSite()
getCurrentCallSite - Get the call site currently being processed, if any.
uint64_t getTypeStoreSize(Type *Ty)
getTypeStoreSize - Return the DataLayout store size for the given type, if known, or a conservative v...
bool isFloatingPointTy() const
isFloatingPointTy - Return true if this is one of the six floating point types
Definition: Type.h:159
bool ShouldEmitAsBranches(const std::vector< CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
Pass structure by value.
Definition: Attributes.h:73
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:659
StoreInst - an instruction for storing to memory.
Definition: Instructions.h:316
static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog2 - Lower a log2 intrinsic.
int getMinimumJumpTableEntries() const
Return integer threshold on number of blocks to use jump tables rather than if sequence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:804
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
bool allowReciprocal() const
Definition: Operator.h:192
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:624
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber.
Definition: InlineAsm.h:120
unsigned getNumElements() const
Return the number of elements in the Vector type.
Definition: DerivedTypes.h:432
static bool isRegDefKind(unsigned Flag)
Definition: InlineAsm.h:315
Debug location.
void setStackProtectorIndex(int I)
iterator begin()
Definition: Function.h:457
bool noInfs() const
Definition: Operator.h:190
void setOrigAlign(unsigned A)
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:142
Type * getElementType() const
Definition: DerivedTypes.h:323
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
Definition: ValueTypes.h:195
bool isAtomic() const
isAtomic - Return true if this instruction has an AtomicOrdering of unordered or higher.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
void AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI, MachineBasicBlock *MBB)
AddLandingPadInfo - Extract the exception handling information from the landingpad instruction and ad...
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
void append(const RegsForValue &RHS)
append - Add the specified values to this one.
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
SDValue getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:436
TargetSelectionDAGInfo - Targets can subclass this to parameterize the SelectionDAG lowering and inst...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:97
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:711
bool unsafeAlgebra() const
Definition: Operator.h:193
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1835
DBG_VALUE - a mapping of the llvm.dbg.value intrinsic.
Definition: TargetOpcodes.h:69
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:57
SDNode * getNode() const
get the SDNode which holds the desired result
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
Definition: Instructions.h:830
bundle_iterator< MachineInstr, instr_iterator > iterator
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
Definition: APFloat.h:122
bool getLibFunc(StringRef funcName, LibFunc::Func &F) const
Searches for a particular function name.
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:245
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
#define P(N)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, SDLoc DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
Instruction that records the offset of a local stack allocation passed to llvm.localescape.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:157
Patchable call instruction - this instruction represents a call to a constant address, followed by a series of NOPs.
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
LandingPadInst - The landingpad instruction holds all of the information necessary to generate correc...
Float from Float.
Definition: ISDOpcodes.h:887
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:533
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:222
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:365
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:35
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:109
bool HasTailCall
HasTailCall - This is set to true if a call in the current block has been translated as a tail call...
unsigned getVectorNumElements() const
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:81
bool hasSideEffects() const
Definition: InlineAsm.h:68
MVT - Machine Value Type.
SynchronizationScope getSynchScope() const
Returns whether this RMW is atomic between threads or only within a single thread.
Definition: Instructions.h:775
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
Definition: InstrTypes.h:62
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:41
DIExpression * getExpression() const
Definition: IntrinsicInst.h:88
void AddInlineAsmOperands(unsigned Kind, bool HasMatching, unsigned MatchingIdx, SDLoc dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
AddInlineAsmOperands - Add this value to the specified inlineasm node operand list.
const Function * getParent() const
Definition: Argument.h:49
Simple binary floating point operators.
Definition: ISDOpcodes.h:237
BranchInst - Conditional or Unconditional Branch instruction.
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
UnreachableInst - This function has undefined behavior.
This is an important base class in LLVM.
Definition: Constant.h:41
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:607
bool isVector() const
isVector - Return true if this is a vector value type.
ResumeInst - Resume the propagation of an exception.
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
IndirectBrInst - Indirect Branch Instruction.
static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl, SelectionDAG &DAG)
void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad=nullptr)
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1895
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned getAlignment() const
getAlignment - Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:130
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:233
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
EmitTargetCodeForMemchr - Emit target-specific code that performs a memchr, in cases where that is fa...
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:332
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed...
Definition: Operator.h:126
This pseudo-instruction loads the stack guard value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Pass structure in an alloca.
Definition: Attributes.h:74
op_iterator op_end()
Definition: User.h:185
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:225
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences ""for some float libcalls"), cl::location(LimitFloatPrecision), cl::init(0))
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, SDLoc dl)
getF32Constant - Get 32-bit floating point constant.
SDValue getCopyFromReg(SDValue Chain, SDLoc dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:547
void visitBitTestCase(BitTestBlock &BB, MachineBasicBlock *NextMBB, uint32_t BranchWeightToNext, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
SDValue getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:635
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
getCopyToRegs - Emit a series of CopyToReg nodes that copies the specified value into the registers s...
void clear()
Clear the memory usage of this object.
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual void EmitFunctionEntryCode()
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
void setNoInfs(bool b)
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:697
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:436
Utility class for integer arithmetic operators which may exhibit overflow - Add, Sub, and Mul.
Definition: Operator.h:74
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1273
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, Type *LoadTy, SelectionDAGBuilder &Builder)
unsigned getOpcode() const
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:644
bool doesNotAccessMemory() const
Determine if the call does not access memory.
Value * getOperand(unsigned i) const
Definition: User.h:118
Zero extended before/after call.
Definition: Attributes.h:119
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandExp2 - Lower an exp2 intrinsic.
Value * getPointerOperand()
Definition: Instructions.h:284
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:647
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
arg_iterator arg_begin()
Definition: Function.h:472
Signed from Signed.
Definition: ISDOpcodes.h:892
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
addInvoke - Provide the begin and end labels of an invoke style call and associate it with a try land...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
ConstantVector - Constant Vector Declarations.
Definition: Constants.h:461
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
Definition: InlineAsm.h:299
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, SDLoc DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:338
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:468
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
unsigned getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:416
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:317
EVT - Extended Value Type.
Definition: ValueTypes.h:31
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
Definition: Type.h:217
std::vector< ArgListEntry > ArgListTy
unsigned getExceptionPointerRegister() const
If a physical register, this returns the register that receives the exception address on entry to a l...
VAArgInst - This class represents the va_arg llvm instruction, which returns an argument of the speci...
SDValue getGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, uint32_t TW, uint32_t FW)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:519
This structure contains all information that is necessary for lowering calls.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, SDLoc, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array...
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:386
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:670
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:335
SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTrunc)
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
unsigned getNumSuccessors() const
The memory access is invariant.
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
void dump() const
Support for debugging, callable in GDB: V->dump()
Definition: AsmWriter.cpp:3353
BasicBlock * getSuccessor(unsigned i) const
TargetIntrinsicInfo - Interface to description of machine instruction set.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it, emit nodes to copy the value into the virtual registers.
Representation for a specific memory location.
virtual const TargetFrameLowering * getFrameLowering() const
std::vector< AsmOperandInfo > AsmOperandInfoVector
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:674
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:304
DIExpression * getExpression() const
Value * getValOperand()
Definition: Instructions.h:783
SDValue getConvertRndSat(EVT VT, SDLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code)
Returns the ConvertRndSat Note: Avoid using this node because it may disappear in the future and most...
SDDbgValue * getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, uint64_t Off, DebugLoc DL, unsigned O)
Constant.
void setNoUnsignedWrap(bool b)
AtomicOrdering getOrdering() const
Returns the ordering effect of this store.
Definition: Instructions.h:372
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:449
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
static PointerType * getUnqual(Type *ElementType)
PointerType::getUnqual - This constructs a pointer to an object of the specified type in the generic ...
Definition: DerivedTypes.h:460
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition: APInt.cpp:552
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:388
const MCContext & getContext() const
const DataLayout * getDataLayout() const
Deprecated in 3.7, will be removed in 3.8.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:216
This is a utility class that provides an abstraction for the common functionality between Instruction...
Definition: Operator.h:32
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
SelectPatternFlavor
Specific patterns of select instructions we can match.
Provides information about what library functions are available for the current target.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MDNode * getMetadata(unsigned KindID) const
getMetadata - Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:167
void clear()
clear - Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuil...
bool isVolatile() const
isVolatile - Return true if this is a load from a volatile memory location.
Definition: Instructions.h:232
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:548
Signed from Float.
Definition: ISDOpcodes.h:890
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
An SDNode that represents everything that will be needed to construct a MachineInstr.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
EmitTargetCodeForMemcmp - Emit target-specific code that performs a memcmp, in cases where that is fa...
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:335
void visit(const Instruction &I)
Unsigned from Signed.
Definition: ISDOpcodes.h:894
unsigned arg_size() const
Definition: CallSite.h:162
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block, return true.
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:458
SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static bool isSequentialInRange(const SmallVectorImpl< int > &Mask, unsigned Pos, unsigned Size, int Low)
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
Definition: Instructions.h:250
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:161
void setNoNaNs(bool b)
Represents one node in the SelectionDAG.
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
const MachineInstrBuilder & addFrameIndex(int Idx) const
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
UpdateSplitBlock - When an MBB was split during scheduling, update the references that need to refer ...
bool isNullValue() const
isNullValue - Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:75
ppc loop data prefetch
static void getCopyToParts(SelectionDAG &DAG, SDLoc DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts...
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:530
BinOp getOperation() const
Definition: Instructions.h:726
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
DWARF expression.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:159
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1: ...
static bool areJTsAllowed(const TargetLowering &TLI)
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:468
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
AttributeSet getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:181
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
SynchronizationScope getSynchScope() const
Definition: Instructions.h:480
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
VectorType - Class to represent vector types.
Definition: DerivedTypes.h:362
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
static const unsigned MaxParallelChains
Class for arbitrary precision integers.
Definition: APInt.h:73
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
BranchProbabilityInfo * BPI
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:342
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
Definition: Type.h:193
This file defines the FastISel class.
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
iterator_range< user_iterator > users()
Definition: Value.h:300
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, ISD::NodeType AssertOp=ISD::DELETED_NODE)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
std::vector< JumpTableBlock > JTCases
JTCases - Vector of JumpTable structures used to communicate SwitchInst code generation information...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
static AttributeSet getReturnAttrs(TargetLowering::CallLoweringInfo &CLI)
Returns an AttributeSet representing the attributes applied to the return value of the given call...
static bool IsOnlyUsedInZeroEqualityComparison(const Value *V)
IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the value is equal or not-eq...
static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI, SDLoc DL, SDISelAsmOperandInfo &OpInfo)
GetRegistersForValue - Assign registers (virtual or physical) for the specified operand.
Value * getCondition() const
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:250
const AttributeSet & getAttributes() const
getAttributes - Return the parameter attributes for this call.
int getStackProtectorIndex() const
Return the index for the stack protector object.
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned char TargetFlags=0)
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
SDDbgValue * getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t Off, DebugLoc DL, unsigned O)
FrameIndex.
const Type * getScalarType() const LLVM_READONLY
getScalarType - If this is a vector type, return the element type, otherwise return 'this'...
Definition: Type.cpp:51
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
SmallVector< SDValue, 8 > PendingLoads
PendingLoads - Loads are not emitted to the program immediately.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
SI Fix SGPR Live Ranges
CallLoweringInfo & setTailCall(bool Value=true)
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:542
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:603
static StringRef getRealLinkageName(StringRef Name)
If special LLVM prefix that is used to inform the asm printer to not emit usual symbol prefix before ...
Definition: GlobalValue.h:306
unsigned countLeadingOnes() const
Count the number of leading one bits.
Definition: APInt.cpp:722
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
cl::opt< std::string > TrapFuncName("trap-func", cl::Hidden, cl::desc("Emit a call to trap function rather than a trap instruction"), cl::init(""))
These are IR-level optimization flags that may be propagated to SDNodes.
Represents a use of a SDNode.
SmallVector< SDValue, 32 > OutVals
bool isLandingPad() const
isLandingPad - Returns true if the block is a landing pad.
static const fltSemantics IEEEsingle
Definition: APFloat.h:132
Value * getCondition() const
Analysis pass providing branch probability information.
BasicBlock * getDefaultDest() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:134
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:217
bool isAggregateType() const
isAggregateType - Return true if the type is an aggregate type.
Definition: Type.h:260
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
Definition: Instructions.h:469
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:128
DbgValueInst - This represents the llvm.dbg.value instruction.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:243
Value * getPointerOperand()
Definition: Instructions.h:779
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
getAAMetadata - Fills the AAMDNodes structure with AA metadata from this instruction.
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:418
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - get or set the calling convention of the call.
Definition: CallSite.h:212
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:311
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:705
iterator end()
Definition: DenseMap.h:68
bool isTailCall() const
std::pair< SDValue, SDValue > lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, MachineBasicBlock *LandingPad=nullptr, bool IsPatchPoint=false)
Lower an argument list according to the target calling convention.
iterator find(const KeyT &Val)
Definition: DenseMap.h:124
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineSDNode * getMachineNode(unsigned Opcode, SDLoc dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
Same for multiplication.
Definition: ISDOpcodes.h:234
unsigned CreateReg(MVT VT)
CreateReg - Allocate a single virtual register for the given type.
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block...
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.setjmp intrinsic.
Definition: ISDOpcodes.h:103
unsigned getNumCases() const
getNumCases - return the number of 'cases' in this switch instruction, except the default case ...
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:166
SDValue getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, const AllocaInst *Alloca=nullptr)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
getCopyFromRegs - If there was virtual register allocated for the value V emit CopyFromReg of the spe...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool hasLocalLinkage() const
Definition: GlobalValue.h:280
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin...
Definition: ISDOpcodes.h:97
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
EmitTargetCodeForStrcmp - Emit target-specific code that performs a strcmp, in cases where that is fa...
Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return the attribute object that exists at the given index.
Definition: Attributes.cpp:985
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
void setUnsafeAlgebra(bool b)
Type * getType() const
getType - Return the type of the instruction that generated this call site
Definition: CallSite.h:166
SwitchInst - Multiway switch.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:279
unsigned getReg() const
getReg - Returns the register number.
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:140
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:105
bool use_empty() const
Definition: Value.h:275
void erase(iterator MBBI)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
SmallVector< MVT, 4 > RegVTs
RegVTs - The value types of the registers.
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:85
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned char TargetFlags=0)
void insert(iterator MBBI, MachineBasicBlock *MBB)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const ARM::ArchExtKind Kind
void setUnusedArgValue(const Value *V, SDValue NewN)
unsigned getParamAlignment(unsigned i) const
Extract the alignment for a call or parameter (0=unknown).
Definition: Function.h:261
bool isEmptyTy() const
isEmptyTy - Return true if this type is empty, that is, it has no elements or all its elements are em...
Definition: Type.cpp:102
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
virtual const TargetInstrInfo * getInstrInfo() const
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVM Value Representation.
Definition: Value.h:69
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:240
SDValue getRegister(unsigned Reg, EVT VT)
static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1023
void init(GCFunctionInfo *gfi, AliasAnalysis &aa, const TargetLibraryInfo *li)
SDValue getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label)
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
Value * getAddress() const
DbgDeclareInst - This represents the llvm.dbg.declare instruction.
DILocalVariable * getVariable() const
void setDebugLoc(DebugLoc dl)
Set source location info.
SDValue getValueType(EVT)
const Value * getArraySize() const
getArraySize - Get the number of elements allocated.
Definition: Instructions.h:110
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:653
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:507
BasicBlockListType::iterator iterator
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:287
const Value * getValue() const
DbgValueInst - This represents the llvm.dbg.value instruction.
InvokeInst - Invoke instruction.
std::vector< CaseBlock > SwitchCases
SwitchCases - Vector of CaseBlock structures used to communicate SwitchInst code generation informati...
#define DEBUG(X)
Definition: Debug.h:92
uint32_t getEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get the raw edge weight calculated for the edge.
IterTy arg_begin() const
arg_begin/arg_end - Return iterators corresponding to the actual argument list for a call site...
Definition: CallSite.h:151
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SDValue getControlRoot()
getControlRoot - Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports items.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Signed from Unsigned.
Definition: ISDOpcodes.h:893
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:164
vt_iterator vt_begin() const
vt_begin / vt_end - Loop over all of the value types that can be represented by values in this regist...
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:40
Garbage collection metadata for a single function.
Definition: GCMetadata.h:73
MachineModuleInfo & getMMI() const
Float from Signed.
Definition: ISDOpcodes.h:888
void clearDanglingDebugInfo()
clearDanglingDebugInfo - Clear the dangling debug information map.
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget=false)
static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI)
visitPow - Lower a pow intrinsic.
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
LPadToCallSiteMap - Map a landing pad to the call site indexes.
unsigned getExceptionSelectorRegister() const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:726
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
MVT getVectorElementType() const
static bool isVolatile(Instruction *Inst)
Conversion operators.
Definition: ISDOpcodes.h:380
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:460
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
Unsigned from Float.
Definition: ISDOpcodes.h:891
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:666
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
bool isUIntN(unsigned N, uint64_t x)
isUIntN - Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:315
Type * getAllocatedType() const
getAllocatedType - Return the type that is being allocated by the instruction.
Definition: Instructions.h:122
void setArgumentFrameIndex(const Argument *A, int FI)
setArgumentFrameIndex - Record frame index for the byval argument.
DenseMap< const Constant *, unsigned > ConstantsOut
bool isAlignStack() const
Definition: InlineAsm.h:69
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void ComputeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo *MMI)
ComputeUsesVAFloatArgument - Determine if any floating-point values are being passed to this variadic...
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
bool isBigEndian() const
Definition: DataLayout.h:218
SynchronizationScope getSynchScope() const
Returns whether this cmpxchg is atomic between threads or only within a single thread.
Definition: Instructions.h:603
SDDbgValue - Holds the information from a dbg_value node through SDISel.
DbgDeclareInst - This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:82
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:506
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
SmallVector< unsigned, 4 > Regs
Regs - This list holds the registers assigned to the values.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.cpp:229
Value * getPointerOperand()
Definition: Instructions.h:409
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
void addSuccessor(MachineBasicBlock *succ, uint32_t weight=0)
addSuccessor - Add succ as a successor of this MachineBasicBlock.
SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope)
Gets a node for an atomic cmpxchg op.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:662
static MachineOperand CreateFI(int Idx)
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests" ...
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:340
const BasicBlock * getParent() const
Definition: Instruction.h:72
bool isExportedInst(const Value *V)
isExportedInst - Return true if the specified value is an instruction exported from its block...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
Float from Unsigned.
Definition: ISDOpcodes.h:889
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:659
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo * FuncInfo
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:203
SmallVector< int, 16 > getShuffleMask() const
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI)
expandLog - Lower a log intrinsic.
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:277
SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget=false)
bool noNaNs() const
Flag queries.
Definition: Operator.h:189
SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align=0, bool Vol=false, bool ReadMem=true, bool WriteMem=true, unsigned Size=0)
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
MachineModuleInfo - This class contains meta information specific to a module.
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
isVoidTy - Return true if this is 'void'.
Definition: Type.h:137
AllocaInst - an instruction to allocate memory on the stack.
Definition: Instructions.h:76
unsigned getLiveInPhysReg(unsigned VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
InsertValueInst - This instruction inserts a struct field of array element value into an aggregate va...
unsigned InitializeRegForValue(const Value *V)
bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:110
BRIND - Indirect branch.
Definition: ISDOpcodes.h:538
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
static bool InBlock(const Value *V, const BasicBlock *BB)
void resize(size_type N)
Definition: SmallVector.h:376
const Use * const_op_iterator
Definition: User.h:179
This class is used to represent ISD::LOAD nodes.
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:527