LLVM  8.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/APInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/None.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/ADT/Triple.h"
29 #include "llvm/ADT/Twine.h"
34 #include "llvm/Analysis/Loads.h"
39 #include "llvm/CodeGen/Analysis.h"
57 #include "llvm/CodeGen/StackMaps.h"
66 #include "llvm/IR/Argument.h"
67 #include "llvm/IR/Attributes.h"
68 #include "llvm/IR/BasicBlock.h"
69 #include "llvm/IR/CFG.h"
70 #include "llvm/IR/CallSite.h"
71 #include "llvm/IR/CallingConv.h"
72 #include "llvm/IR/Constant.h"
73 #include "llvm/IR/ConstantRange.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/InlineAsm.h"
82 #include "llvm/IR/InstrTypes.h"
83 #include "llvm/IR/Instruction.h"
84 #include "llvm/IR/Instructions.h"
85 #include "llvm/IR/IntrinsicInst.h"
86 #include "llvm/IR/Intrinsics.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Metadata.h"
89 #include "llvm/IR/Module.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/Statepoint.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCSymbol.h"
99 #include "llvm/Support/Casting.h"
100 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/Compiler.h"
103 #include "llvm/Support/Debug.h"
106 #include "llvm/Support/MathExtras.h"
111 #include <algorithm>
112 #include <cassert>
113 #include <cstddef>
114 #include <cstdint>
115 #include <cstring>
116 #include <iterator>
117 #include <limits>
118 #include <numeric>
119 #include <tuple>
120 #include <utility>
121 #include <vector>
122 
123 using namespace llvm;
124 
125 #define DEBUG_TYPE "isel"
126 
127 /// LimitFloatPrecision - Generate low-precision inline sequences for
128 /// some float libcalls (6, 8 or 12 bits).
129 static unsigned LimitFloatPrecision;
130 
132  LimitFPPrecision("limit-float-precision",
133  cl::desc("Generate low-precision inline sequences "
134  "for some float libcalls"),
135  cl::location(LimitFloatPrecision), cl::Hidden,
136  cl::init(0));
137 
139  "switch-peel-threshold", cl::Hidden, cl::init(66),
140  cl::desc("Set the case probability threshold for peeling the case from a "
141  "switch statement. A value greater than 100 will void this "
142  "optimization"));
143 
144 // Limit the width of DAG chains. This is important in general to prevent
145 // DAG-based analysis from blowing up. For example, alias analysis and
146 // load clustering may not complete in reasonable time. It is difficult to
147 // recognize and avoid this situation within each individual analysis, and
148 // future analyses are likely to have the same behavior. Limiting DAG width is
149 // the safe approach and will be especially important with global DAGs.
150 //
151 // MaxParallelChains default is arbitrarily high to avoid affecting
152 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
153 // sequence over this should have been converted to llvm.memcpy by the
154 // frontend. It is easy to induce this behavior with .ll code such as:
155 // %buffer = alloca [4096 x i8]
156 // %data = load [4096 x i8]* %argPtr
157 // store [4096 x i8] %data, [4096 x i8]* %buffer
158 static const unsigned MaxParallelChains = 64;
159 
160 // Return the calling convention if the Value passed requires ABI mangling as it
161 // is a parameter to a function or a return value from a function which is not
162 // an intrinsic.
164  if (auto *R = dyn_cast<ReturnInst>(V))
165  return R->getParent()->getParent()->getCallingConv();
166 
167  if (auto *CI = dyn_cast<CallInst>(V)) {
168  const bool IsInlineAsm = CI->isInlineAsm();
169  const bool IsIndirectFunctionCall =
170  !IsInlineAsm && !CI->getCalledFunction();
171 
172  // It is possible that the call instruction is an inline asm statement or an
173  // indirect function call in which case the return value of
174  // getCalledFunction() would be nullptr.
175  const bool IsInstrinsicCall =
176  !IsInlineAsm && !IsIndirectFunctionCall &&
177  CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
178 
179  if (!IsInlineAsm && !IsInstrinsicCall)
180  return CI->getCallingConv();
181  }
182 
183  return None;
184 }
185 
186 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
187  const SDValue *Parts, unsigned NumParts,
188  MVT PartVT, EVT ValueVT, const Value *V,
190 
191 /// getCopyFromParts - Create a value that contains the specified legal parts
192 /// combined into the value they represent. If the parts combine to a type
193 /// larger than ValueVT then AssertOp can be used to specify whether the extra
194 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
195 /// (ISD::AssertSext).
197  const SDValue *Parts, unsigned NumParts,
198  MVT PartVT, EVT ValueVT, const Value *V,
200  Optional<ISD::NodeType> AssertOp = None) {
201  if (ValueVT.isVector())
202  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
203  CC);
204 
205  assert(NumParts > 0 && "No parts to assemble!");
206  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
207  SDValue Val = Parts[0];
208 
209  if (NumParts > 1) {
210  // Assemble the value from multiple parts.
211  if (ValueVT.isInteger()) {
212  unsigned PartBits = PartVT.getSizeInBits();
213  unsigned ValueBits = ValueVT.getSizeInBits();
214 
215  // Assemble the power of 2 part.
216  unsigned RoundParts = NumParts & (NumParts - 1) ?
217  1 << Log2_32(NumParts) : NumParts;
218  unsigned RoundBits = PartBits * RoundParts;
219  EVT RoundVT = RoundBits == ValueBits ?
220  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
221  SDValue Lo, Hi;
222 
223  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
224 
225  if (RoundParts > 2) {
226  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
227  PartVT, HalfVT, V);
228  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
229  RoundParts / 2, PartVT, HalfVT, V);
230  } else {
231  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
232  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
233  }
234 
235  if (DAG.getDataLayout().isBigEndian())
236  std::swap(Lo, Hi);
237 
238  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
239 
240  if (RoundParts < NumParts) {
241  // Assemble the trailing non-power-of-2 part.
242  unsigned OddParts = NumParts - RoundParts;
243  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
244  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
245  OddVT, V, CC);
246 
247  // Combine the round and odd parts.
248  Lo = Val;
249  if (DAG.getDataLayout().isBigEndian())
250  std::swap(Lo, Hi);
251  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
252  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
253  Hi =
254  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
255  DAG.getConstant(Lo.getValueSizeInBits(), DL,
256  TLI.getPointerTy(DAG.getDataLayout())));
257  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
258  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
259  }
260  } else if (PartVT.isFloatingPoint()) {
261  // FP split into multiple FP parts (for ppcf128)
262  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
263  "Unexpected split");
264  SDValue Lo, Hi;
265  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
266  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
267  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
268  std::swap(Lo, Hi);
269  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
270  } else {
271  // FP split into integer parts (soft fp)
272  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
273  !PartVT.isVector() && "Unexpected split");
274  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
275  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
276  }
277  }
278 
279  // There is now one part, held in Val. Correct it to match ValueVT.
280  // PartEVT is the type of the register class that holds the value.
281  // ValueVT is the type of the inline asm operation.
282  EVT PartEVT = Val.getValueType();
283 
284  if (PartEVT == ValueVT)
285  return Val;
286 
287  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
288  ValueVT.bitsLT(PartEVT)) {
289  // For an FP value in an integer part, we need to truncate to the right
290  // width first.
291  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
292  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
293  }
294 
295  // Handle types that have the same size.
296  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
297  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
298 
299  // Handle types with different sizes.
300  if (PartEVT.isInteger() && ValueVT.isInteger()) {
301  if (ValueVT.bitsLT(PartEVT)) {
302  // For a truncate, see if we have any information to
303  // indicate whether the truncated bits will always be
304  // zero or sign-extension.
305  if (AssertOp.hasValue())
306  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
307  DAG.getValueType(ValueVT));
308  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
309  }
310  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
311  }
312 
313  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
314  // FP_ROUND's are always exact here.
315  if (ValueVT.bitsLT(Val.getValueType()))
316  return DAG.getNode(
317  ISD::FP_ROUND, DL, ValueVT, Val,
318  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
319 
320  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
321  }
322 
323  llvm_unreachable("Unknown mismatch!");
324 }
325 
327  const Twine &ErrMsg) {
328  const Instruction *I = dyn_cast_or_null<Instruction>(V);
329  if (!V)
330  return Ctx.emitError(ErrMsg);
331 
332  const char *AsmError = ", possible invalid constraint for vector type";
333  if (const CallInst *CI = dyn_cast<CallInst>(I))
334  if (isa<InlineAsm>(CI->getCalledValue()))
335  return Ctx.emitError(I, ErrMsg + AsmError);
336 
337  return Ctx.emitError(I, ErrMsg);
338 }
339 
340 /// getCopyFromPartsVector - Create a value that contains the specified legal
341 /// parts combined into the value they represent. If the parts combine to a
342 /// type larger than ValueVT then AssertOp can be used to specify whether the
343 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
344 /// ValueVT (ISD::AssertSext).
346  const SDValue *Parts, unsigned NumParts,
347  MVT PartVT, EVT ValueVT, const Value *V,
348  Optional<CallingConv::ID> CallConv) {
349  assert(ValueVT.isVector() && "Not a vector value");
350  assert(NumParts > 0 && "No parts to assemble!");
351  const bool IsABIRegCopy = CallConv.hasValue();
352 
353  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
354  SDValue Val = Parts[0];
355 
356  // Handle a multi-element vector.
357  if (NumParts > 1) {
358  EVT IntermediateVT;
359  MVT RegisterVT;
360  unsigned NumIntermediates;
361  unsigned NumRegs;
362 
363  if (IsABIRegCopy) {
365  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
366  NumIntermediates, RegisterVT);
367  } else {
368  NumRegs =
369  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
370  NumIntermediates, RegisterVT);
371  }
372 
373  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
374  NumParts = NumRegs; // Silence a compiler warning.
375  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
376  assert(RegisterVT.getSizeInBits() ==
377  Parts[0].getSimpleValueType().getSizeInBits() &&
378  "Part type sizes don't match!");
379 
380  // Assemble the parts into intermediate operands.
381  SmallVector<SDValue, 8> Ops(NumIntermediates);
382  if (NumIntermediates == NumParts) {
383  // If the register was not expanded, truncate or copy the value,
384  // as appropriate.
385  for (unsigned i = 0; i != NumParts; ++i)
386  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
387  PartVT, IntermediateVT, V);
388  } else if (NumParts > 0) {
389  // If the intermediate type was expanded, build the intermediate
390  // operands from the parts.
391  assert(NumParts % NumIntermediates == 0 &&
392  "Must expand into a divisible number of parts!");
393  unsigned Factor = NumParts / NumIntermediates;
394  for (unsigned i = 0; i != NumIntermediates; ++i)
395  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
396  PartVT, IntermediateVT, V);
397  }
398 
399  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
400  // intermediate operands.
401  EVT BuiltVectorTy =
402  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
403  (IntermediateVT.isVector()
404  ? IntermediateVT.getVectorNumElements() * NumParts
405  : NumIntermediates));
406  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
408  DL, BuiltVectorTy, Ops);
409  }
410 
411  // There is now one part, held in Val. Correct it to match ValueVT.
412  EVT PartEVT = Val.getValueType();
413 
414  if (PartEVT == ValueVT)
415  return Val;
416 
417  if (PartEVT.isVector()) {
418  // If the element type of the source/dest vectors are the same, but the
419  // parts vector has more elements than the value vector, then we have a
420  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
421  // elements we want.
422  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
423  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
424  "Cannot narrow, it would be a lossy transformation");
425  return DAG.getNode(
426  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
427  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
428  }
429 
430  // Vector/Vector bitcast.
431  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433 
434  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
435  "Cannot handle this kind of promotion");
436  // Promoted vector extract
437  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
438 
439  }
440 
441  // Trivial bitcast if the types are the same size and the destination
442  // vector type is legal.
443  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
444  TLI.isTypeLegal(ValueVT))
445  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
446 
447  if (ValueVT.getVectorNumElements() != 1) {
448  // Certain ABIs require that vectors are passed as integers. For vectors
449  // are the same size, this is an obvious bitcast.
450  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
451  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
453  // Bitcast Val back the original type and extract the corresponding
454  // vector we want.
455  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
456  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
457  ValueVT.getVectorElementType(), Elts);
458  Val = DAG.getBitcast(WiderVecType, Val);
459  return DAG.getNode(
460  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
461  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
462  }
463 
465  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
466  return DAG.getUNDEF(ValueVT);
467  }
468 
469  // Handle cases such as i8 -> <1 x i1>
470  EVT ValueSVT = ValueVT.getVectorElementType();
471  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
472  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
473  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
474 
475  return DAG.getBuildVector(ValueVT, DL, Val);
476 }
477 
478 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
479  SDValue Val, SDValue *Parts, unsigned NumParts,
480  MVT PartVT, const Value *V,
481  Optional<CallingConv::ID> CallConv);
482 
483 /// getCopyToParts - Create a series of nodes that contain the specified value
484 /// split into legal parts. If the parts contain more bits than Val, then, for
485 /// integers, ExtendKind can be used to specify how to generate the extra bits.
486 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
487  SDValue *Parts, unsigned NumParts, MVT PartVT,
488  const Value *V,
489  Optional<CallingConv::ID> CallConv = None,
490  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
491  EVT ValueVT = Val.getValueType();
492 
493  // Handle the vector case separately.
494  if (ValueVT.isVector())
495  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
496  CallConv);
497 
498  unsigned PartBits = PartVT.getSizeInBits();
499  unsigned OrigNumParts = NumParts;
500  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
501  "Copying to an illegal type!");
502 
503  if (NumParts == 0)
504  return;
505 
506  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
507  EVT PartEVT = PartVT;
508  if (PartEVT == ValueVT) {
509  assert(NumParts == 1 && "No-op copy with multiple parts!");
510  Parts[0] = Val;
511  return;
512  }
513 
514  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
515  // If the parts cover more bits than the value has, promote the value.
516  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
517  assert(NumParts == 1 && "Do not know what to promote to!");
518  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
519  } else {
520  if (ValueVT.isFloatingPoint()) {
521  // FP values need to be bitcast, then extended if they are being put
522  // into a larger container.
523  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
524  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
525  }
526  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
527  ValueVT.isInteger() &&
528  "Unknown mismatch!");
529  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
530  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
531  if (PartVT == MVT::x86mmx)
532  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533  }
534  } else if (PartBits == ValueVT.getSizeInBits()) {
535  // Different types of the same size.
536  assert(NumParts == 1 && PartEVT != ValueVT);
537  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
538  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
539  // If the parts cover less bits than value has, truncate the value.
540  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
541  ValueVT.isInteger() &&
542  "Unknown mismatch!");
543  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
544  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
545  if (PartVT == MVT::x86mmx)
546  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547  }
548 
549  // The value may have changed - recompute ValueVT.
550  ValueVT = Val.getValueType();
551  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
552  "Failed to tile the value with PartVT!");
553 
554  if (NumParts == 1) {
555  if (PartEVT != ValueVT) {
557  "scalar-to-vector conversion failed");
558  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559  }
560 
561  Parts[0] = Val;
562  return;
563  }
564 
565  // Expand the value into multiple parts.
566  if (NumParts & (NumParts - 1)) {
567  // The number of parts is not a power of 2. Split off and copy the tail.
568  assert(PartVT.isInteger() && ValueVT.isInteger() &&
569  "Do not know what to expand to!");
570  unsigned RoundParts = 1 << Log2_32(NumParts);
571  unsigned RoundBits = RoundParts * PartBits;
572  unsigned OddParts = NumParts - RoundParts;
573  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
574  DAG.getIntPtrConstant(RoundBits, DL));
575  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
576  CallConv);
577 
578  if (DAG.getDataLayout().isBigEndian())
579  // The odd parts were reversed by getCopyToParts - unreverse them.
580  std::reverse(Parts + RoundParts, Parts + NumParts);
581 
582  NumParts = RoundParts;
583  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
584  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
585  }
586 
587  // The number of parts is a power of 2. Repeatedly bisect the value using
588  // EXTRACT_ELEMENT.
589  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
591  ValueVT.getSizeInBits()),
592  Val);
593 
594  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
595  for (unsigned i = 0; i < NumParts; i += StepSize) {
596  unsigned ThisBits = StepSize * PartBits / 2;
597  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
598  SDValue &Part0 = Parts[i];
599  SDValue &Part1 = Parts[i+StepSize/2];
600 
601  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
602  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
603  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
605 
606  if (ThisBits == PartBits && ThisVT != PartVT) {
607  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
608  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
609  }
610  }
611  }
612 
613  if (DAG.getDataLayout().isBigEndian())
614  std::reverse(Parts, Parts + OrigNumParts);
615 }
616 
618  SDValue Val, const SDLoc &DL, EVT PartVT) {
619  if (!PartVT.isVector())
620  return SDValue();
621 
622  EVT ValueVT = Val.getValueType();
623  unsigned PartNumElts = PartVT.getVectorNumElements();
624  unsigned ValueNumElts = ValueVT.getVectorNumElements();
625  if (PartNumElts > ValueNumElts &&
626  PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
627  EVT ElementVT = PartVT.getVectorElementType();
628  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
629  // undef elements.
631  DAG.ExtractVectorElements(Val, Ops);
632  SDValue EltUndef = DAG.getUNDEF(ElementVT);
633  for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
634  Ops.push_back(EltUndef);
635 
636  // FIXME: Use CONCAT for 2x -> 4x.
637  return DAG.getBuildVector(PartVT, DL, Ops);
638  }
639 
640  return SDValue();
641 }
642 
643 /// getCopyToPartsVector - Create a series of nodes that contain the specified
644 /// value split into legal parts.
645 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
646  SDValue Val, SDValue *Parts, unsigned NumParts,
647  MVT PartVT, const Value *V,
648  Optional<CallingConv::ID> CallConv) {
649  EVT ValueVT = Val.getValueType();
650  assert(ValueVT.isVector() && "Not a vector");
651  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
652  const bool IsABIRegCopy = CallConv.hasValue();
653 
654  if (NumParts == 1) {
655  EVT PartEVT = PartVT;
656  if (PartEVT == ValueVT) {
657  // Nothing to do.
658  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
659  // Bitconvert vector->vector case.
660  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
661  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
662  Val = Widened;
663  } else if (PartVT.isVector() &&
664  PartEVT.getVectorElementType().bitsGE(
665  ValueVT.getVectorElementType()) &&
666  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
667 
668  // Promoted vector extract
669  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
670  } else {
671  if (ValueVT.getVectorNumElements() == 1) {
672  Val = DAG.getNode(
673  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
674  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
675  } else {
676  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
677  "lossy conversion of vector to scalar type");
678  EVT IntermediateType =
679  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
680  Val = DAG.getBitcast(IntermediateType, Val);
681  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
682  }
683  }
684 
685  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
686  Parts[0] = Val;
687  return;
688  }
689 
690  // Handle a multi-element vector.
691  EVT IntermediateVT;
692  MVT RegisterVT;
693  unsigned NumIntermediates;
694  unsigned NumRegs;
695  if (IsABIRegCopy) {
696  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
697  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
698  NumIntermediates, RegisterVT);
699  } else {
700  NumRegs =
701  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
702  NumIntermediates, RegisterVT);
703  }
704 
705  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
706  NumParts = NumRegs; // Silence a compiler warning.
707  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
708 
709  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
710  IntermediateVT.getVectorNumElements() : 1;
711 
712  // Convert the vector to the appropiate type if necessary.
713  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
714 
715  EVT BuiltVectorTy = EVT::getVectorVT(
716  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
717  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
718  if (ValueVT != BuiltVectorTy) {
719  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
720  Val = Widened;
721 
722  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
723  }
724 
725  // Split the vector into intermediate operands.
726  SmallVector<SDValue, 8> Ops(NumIntermediates);
727  for (unsigned i = 0; i != NumIntermediates; ++i) {
728  if (IntermediateVT.isVector()) {
729  Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
730  DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
731  } else {
732  Ops[i] = DAG.getNode(
733  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
734  DAG.getConstant(i, DL, IdxVT));
735  }
736  }
737 
738  // Split the intermediate operands into legal parts.
739  if (NumParts == NumIntermediates) {
740  // If the register was not expanded, promote or copy the value,
741  // as appropriate.
742  for (unsigned i = 0; i != NumParts; ++i)
743  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
744  } else if (NumParts > 0) {
745  // If the intermediate type was expanded, split each the value into
746  // legal parts.
747  assert(NumIntermediates != 0 && "division by zero");
748  assert(NumParts % NumIntermediates == 0 &&
749  "Must expand into a divisible number of parts!");
750  unsigned Factor = NumParts / NumIntermediates;
751  for (unsigned i = 0; i != NumIntermediates; ++i)
752  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
753  CallConv);
754  }
755 }
756 
758  EVT valuevt, Optional<CallingConv::ID> CC)
759  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
760  RegCount(1, regs.size()), CallConv(CC) {}
761 
763  const DataLayout &DL, unsigned Reg, Type *Ty,
765  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
766 
767  CallConv = CC;
768 
769  for (EVT ValueVT : ValueVTs) {
770  unsigned NumRegs =
771  isABIMangled()
772  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
773  : TLI.getNumRegisters(Context, ValueVT);
774  MVT RegisterVT =
775  isABIMangled()
776  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
777  : TLI.getRegisterType(Context, ValueVT);
778  for (unsigned i = 0; i != NumRegs; ++i)
779  Regs.push_back(Reg + i);
780  RegVTs.push_back(RegisterVT);
781  RegCount.push_back(NumRegs);
782  Reg += NumRegs;
783  }
784 }
785 
787  FunctionLoweringInfo &FuncInfo,
788  const SDLoc &dl, SDValue &Chain,
789  SDValue *Flag, const Value *V) const {
790  // A Value with type {} or [0 x %t] needs no registers.
791  if (ValueVTs.empty())
792  return SDValue();
793 
794  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
795 
796  // Assemble the legal parts into the final values.
797  SmallVector<SDValue, 4> Values(ValueVTs.size());
799  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
800  // Copy the legal parts from the registers.
801  EVT ValueVT = ValueVTs[Value];
802  unsigned NumRegs = RegCount[Value];
803  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
804  *DAG.getContext(),
805  CallConv.getValue(), RegVTs[Value])
806  : RegVTs[Value];
807 
808  Parts.resize(NumRegs);
809  for (unsigned i = 0; i != NumRegs; ++i) {
810  SDValue P;
811  if (!Flag) {
812  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
813  } else {
814  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
815  *Flag = P.getValue(2);
816  }
817 
818  Chain = P.getValue(1);
819  Parts[i] = P;
820 
821  // If the source register was virtual and if we know something about it,
822  // add an assert node.
824  !RegisterVT.isInteger() || RegisterVT.isVector())
825  continue;
826 
828  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
829  if (!LOI)
830  continue;
831 
832  unsigned RegSize = RegisterVT.getSizeInBits();
833  unsigned NumSignBits = LOI->NumSignBits;
834  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
835 
836  if (NumZeroBits == RegSize) {
837  // The current value is a zero.
838  // Explicitly express that as it would be easier for
839  // optimizations to kick in.
840  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
841  continue;
842  }
843 
844  // FIXME: We capture more information than the dag can represent. For
845  // now, just use the tightest assertzext/assertsext possible.
846  bool isSExt;
847  EVT FromVT(MVT::Other);
848  if (NumZeroBits) {
849  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
850  isSExt = false;
851  } else if (NumSignBits > 1) {
852  FromVT =
853  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
854  isSExt = true;
855  } else {
856  continue;
857  }
858  // Add an assertion node.
859  assert(FromVT != MVT::Other);
860  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
861  RegisterVT, P, DAG.getValueType(FromVT));
862  }
863 
864  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
865  RegisterVT, ValueVT, V, CallConv);
866  Part += NumRegs;
867  Parts.clear();
868  }
869 
870  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
871 }
872 
874  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
875  const Value *V,
876  ISD::NodeType PreferredExtendType) const {
877  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
878  ISD::NodeType ExtendKind = PreferredExtendType;
879 
880  // Get the list of the values's legal parts.
881  unsigned NumRegs = Regs.size();
882  SmallVector<SDValue, 8> Parts(NumRegs);
883  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
884  unsigned NumParts = RegCount[Value];
885 
886  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
887  *DAG.getContext(),
888  CallConv.getValue(), RegVTs[Value])
889  : RegVTs[Value];
890 
891  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
892  ExtendKind = ISD::ZERO_EXTEND;
893 
894  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
895  NumParts, RegisterVT, V, CallConv, ExtendKind);
896  Part += NumParts;
897  }
898 
899  // Copy the parts into the registers.
900  SmallVector<SDValue, 8> Chains(NumRegs);
901  for (unsigned i = 0; i != NumRegs; ++i) {
902  SDValue Part;
903  if (!Flag) {
904  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
905  } else {
906  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
907  *Flag = Part.getValue(1);
908  }
909 
910  Chains[i] = Part.getValue(0);
911  }
912 
913  if (NumRegs == 1 || Flag)
914  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
915  // flagged to it. That is the CopyToReg nodes and the user are considered
916  // a single scheduling unit. If we create a TokenFactor and return it as
917  // chain, then the TokenFactor is both a predecessor (operand) of the
918  // user as well as a successor (the TF operands are flagged to the user).
919  // c1, f1 = CopyToReg
920  // c2, f2 = CopyToReg
921  // c3 = TokenFactor c1, c2
922  // ...
923  // = op c3, ..., f2
924  Chain = Chains[NumRegs-1];
925  else
926  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
927 }
928 
929 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
930  unsigned MatchingIdx, const SDLoc &dl,
931  SelectionDAG &DAG,
932  std::vector<SDValue> &Ops) const {
933  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
934 
935  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
936  if (HasMatching)
937  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
938  else if (!Regs.empty() &&
940  // Put the register class of the virtual registers in the flag word. That
941  // way, later passes can recompute register class constraints for inline
942  // assembly as well as normal instructions.
943  // Don't do this for tied operands that can use the regclass information
944  // from the def.
946  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
947  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
948  }
949 
950  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
951  Ops.push_back(Res);
952 
953  if (Code == InlineAsm::Kind_Clobber) {
954  // Clobbers should always have a 1:1 mapping with registers, and may
955  // reference registers that have illegal (e.g. vector) types. Hence, we
956  // shouldn't try to apply any sort of splitting logic to them.
957  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
958  "No 1:1 mapping from clobbers to regs?");
959  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
960  (void)SP;
961  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
962  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
963  assert(
964  (Regs[I] != SP ||
966  "If we clobbered the stack pointer, MFI should know about it.");
967  }
968  return;
969  }
970 
971  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
972  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
973  MVT RegisterVT = RegVTs[Value];
974  for (unsigned i = 0; i != NumRegs; ++i) {
975  assert(Reg < Regs.size() && "Mismatch in # registers expected");
976  unsigned TheReg = Regs[Reg++];
977  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
978  }
979  }
980 }
981 
985  unsigned I = 0;
986  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
987  unsigned RegCount = std::get<0>(CountAndVT);
988  MVT RegisterVT = std::get<1>(CountAndVT);
989  unsigned RegisterSize = RegisterVT.getSizeInBits();
990  for (unsigned E = I + RegCount; I != E; ++I)
991  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
992  }
993  return OutVec;
994 }
995 
997  const TargetLibraryInfo *li) {
998  AA = aa;
999  GFI = gfi;
1000  LibInfo = li;
1001  DL = &DAG.getDataLayout();
1002  Context = DAG.getContext();
1003  LPadToCallSiteMap.clear();
1004 }
1005 
1007  NodeMap.clear();
1008  UnusedArgNodeMap.clear();
1009  PendingLoads.clear();
1010  PendingExports.clear();
1011  CurInst = nullptr;
1012  HasTailCall = false;
1013  SDNodeOrder = LowestSDNodeOrder;
1014  StatepointLowering.clear();
1015 }
1016 
1018  DanglingDebugInfoMap.clear();
1019 }
1020 
1022  if (PendingLoads.empty())
1023  return DAG.getRoot();
1024 
1025  if (PendingLoads.size() == 1) {
1026  SDValue Root = PendingLoads[0];
1027  DAG.setRoot(Root);
1028  PendingLoads.clear();
1029  return Root;
1030  }
1031 
1032  // Otherwise, we have to make a token factor node.
1033  SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1034  PendingLoads);
1035  PendingLoads.clear();
1036  DAG.setRoot(Root);
1037  return Root;
1038 }
1039 
1041  SDValue Root = DAG.getRoot();
1042 
1043  if (PendingExports.empty())
1044  return Root;
1045 
1046  // Turn all of the CopyToReg chains into one factored node.
1047  if (Root.getOpcode() != ISD::EntryToken) {
1048  unsigned i = 0, e = PendingExports.size();
1049  for (; i != e; ++i) {
1050  assert(PendingExports[i].getNode()->getNumOperands() > 1);
1051  if (PendingExports[i].getNode()->getOperand(0) == Root)
1052  break; // Don't add the root if we already indirectly depend on it.
1053  }
1054 
1055  if (i == e)
1056  PendingExports.push_back(Root);
1057  }
1058 
1059  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1060  PendingExports);
1061  PendingExports.clear();
1062  DAG.setRoot(Root);
1063  return Root;
1064 }
1065 
1067  // Set up outgoing PHI node register values before emitting the terminator.
1068  if (I.isTerminator()) {
1069  HandlePHINodesInSuccessorBlocks(I.getParent());
1070  }
1071 
1072  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1073  if (!isa<DbgInfoIntrinsic>(I))
1074  ++SDNodeOrder;
1075 
1076  CurInst = &I;
1077 
1078  visit(I.getOpcode(), I);
1079 
1080  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1081  // Propagate the fast-math-flags of this IR instruction to the DAG node that
1082  // maps to this instruction.
1083  // TODO: We could handle all flags (nsw, etc) here.
1084  // TODO: If an IR instruction maps to >1 node, only the final node will have
1085  // flags set.
1086  if (SDNode *Node = getNodeForIRValue(&I)) {
1087  SDNodeFlags IncomingFlags;
1088  IncomingFlags.copyFMF(*FPMO);
1089  if (!Node->getFlags().isDefined())
1090  Node->setFlags(IncomingFlags);
1091  else
1092  Node->intersectFlagsWith(IncomingFlags);
1093  }
1094  }
1095 
1096  if (!I.isTerminator() && !HasTailCall &&
1097  !isStatepoint(&I)) // statepoints handle their exports internally
1098  CopyToExportRegsIfNeeded(&I);
1099 
1100  CurInst = nullptr;
1101 }
1102 
1103 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1104  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1105 }
1106 
1107 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1108  // Note: this doesn't use InstVisitor, because it has to work with
1109  // ConstantExpr's in addition to instructions.
1110  switch (Opcode) {
1111  default: llvm_unreachable("Unknown instruction type encountered!");
1112  // Build the switch statement using the Instruction.def file.
1113 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1114  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1115 #include "llvm/IR/Instruction.def"
1116  }
1117 }
1118 
1120  const DIExpression *Expr) {
1121  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1122  const DbgValueInst *DI = DDI.getDI();
1123  DIVariable *DanglingVariable = DI->getVariable();
1124  DIExpression *DanglingExpr = DI->getExpression();
1125  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1126  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1127  return true;
1128  }
1129  return false;
1130  };
1131 
1132  for (auto &DDIMI : DanglingDebugInfoMap) {
1133  DanglingDebugInfoVector &DDIV = DDIMI.second;
1134  DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1135  }
1136 }
1137 
1138 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1139 // generate the debug data structures now that we've seen its definition.
1141  SDValue Val) {
1142  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1143  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1144  return;
1145 
1146  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1147  for (auto &DDI : DDIV) {
1148  const DbgValueInst *DI = DDI.getDI();
1149  assert(DI && "Ill-formed DanglingDebugInfo");
1150  DebugLoc dl = DDI.getdl();
1151  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1152  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1153  DILocalVariable *Variable = DI->getVariable();
1154  DIExpression *Expr = DI->getExpression();
1155  assert(Variable->isValidLocationForIntrinsic(dl) &&
1156  "Expected inlined-at fields to agree");
1157  SDDbgValue *SDV;
1158  if (Val.getNode()) {
1159  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1160  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1161  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1162  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1163  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1164  // inserted after the definition of Val when emitting the instructions
1165  // after ISel. An alternative could be to teach
1166  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1167  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1168  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1169  << ValSDNodeOrder << "\n");
1170  SDV = getDbgValue(Val, Variable, Expr, dl,
1171  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1172  DAG.AddDbgValue(SDV, Val.getNode(), false);
1173  } else
1174  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1175  << "in EmitFuncArgumentDbgValue\n");
1176  } else
1177  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1178  }
1179  DDIV.clear();
1180 }
1181 
1182 /// getCopyFromRegs - If there was virtual register allocated for the value V
1183 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1185  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1186  SDValue Result;
1187 
1188  if (It != FuncInfo.ValueMap.end()) {
1189  unsigned InReg = It->second;
1190 
1191  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1192  DAG.getDataLayout(), InReg, Ty,
1193  None); // This is not an ABI copy.
1194  SDValue Chain = DAG.getEntryNode();
1195  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1196  V);
1197  resolveDanglingDebugInfo(V, Result);
1198  }
1199 
1200  return Result;
1201 }
1202 
1203 /// getValue - Return an SDValue for the given Value.
1205  // If we already have an SDValue for this value, use it. It's important
1206  // to do this first, so that we don't create a CopyFromReg if we already
1207  // have a regular SDValue.
1208  SDValue &N = NodeMap[V];
1209  if (N.getNode()) return N;
1210 
1211  // If there's a virtual register allocated and initialized for this
1212  // value, use it.
1213  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1214  return copyFromReg;
1215 
1216  // Otherwise create a new SDValue and remember it.
1217  SDValue Val = getValueImpl(V);
1218  NodeMap[V] = Val;
1219  resolveDanglingDebugInfo(V, Val);
1220  return Val;
1221 }
1222 
1223 // Return true if SDValue exists for the given Value
1225  return (NodeMap.find(V) != NodeMap.end()) ||
1226  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1227 }
1228 
1229 /// getNonRegisterValue - Return an SDValue for the given Value, but
1230 /// don't look in FuncInfo.ValueMap for a virtual register.
1232  // If we already have an SDValue for this value, use it.
1233  SDValue &N = NodeMap[V];
1234  if (N.getNode()) {
1235  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1236  // Remove the debug location from the node as the node is about to be used
1237  // in a location which may differ from the original debug location. This
1238  // is relevant to Constant and ConstantFP nodes because they can appear
1239  // as constant expressions inside PHI nodes.
1240  N->setDebugLoc(DebugLoc());
1241  }
1242  return N;
1243  }
1244 
1245  // Otherwise create a new SDValue and remember it.
1246  SDValue Val = getValueImpl(V);
1247  NodeMap[V] = Val;
1248  resolveDanglingDebugInfo(V, Val);
1249  return Val;
1250 }
1251 
1252 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1253 /// Create an SDValue for the given value.
1255  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1256 
1257  if (const Constant *C = dyn_cast<Constant>(V)) {
1258  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1259 
1260  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1261  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1262 
1263  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1264  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1265 
1266  if (isa<ConstantPointerNull>(C)) {
1267  unsigned AS = V->getType()->getPointerAddressSpace();
1268  return DAG.getConstant(0, getCurSDLoc(),
1269  TLI.getPointerTy(DAG.getDataLayout(), AS));
1270  }
1271 
1272  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1273  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1274 
1275  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1276  return DAG.getUNDEF(VT);
1277 
1278  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1279  visit(CE->getOpcode(), *CE);
1280  SDValue N1 = NodeMap[V];
1281  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1282  return N1;
1283  }
1284 
1285  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1287  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1288  OI != OE; ++OI) {
1289  SDNode *Val = getValue(*OI).getNode();
1290  // If the operand is an empty aggregate, there are no values.
1291  if (!Val) continue;
1292  // Add each leaf value from the operand to the Constants list
1293  // to form a flattened list of all the values.
1294  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1295  Constants.push_back(SDValue(Val, i));
1296  }
1297 
1298  return DAG.getMergeValues(Constants, getCurSDLoc());
1299  }
1300 
1301  if (const ConstantDataSequential *CDS =
1302  dyn_cast<ConstantDataSequential>(C)) {
1304  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1305  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1306  // Add each leaf value from the operand to the Constants list
1307  // to form a flattened list of all the values.
1308  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1309  Ops.push_back(SDValue(Val, i));
1310  }
1311 
1312  if (isa<ArrayType>(CDS->getType()))
1313  return DAG.getMergeValues(Ops, getCurSDLoc());
1314  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1315  }
1316 
1317  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1318  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1319  "Unknown struct or array constant!");
1320 
1322  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1323  unsigned NumElts = ValueVTs.size();
1324  if (NumElts == 0)
1325  return SDValue(); // empty struct
1327  for (unsigned i = 0; i != NumElts; ++i) {
1328  EVT EltVT = ValueVTs[i];
1329  if (isa<UndefValue>(C))
1330  Constants[i] = DAG.getUNDEF(EltVT);
1331  else if (EltVT.isFloatingPoint())
1332  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1333  else
1334  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1335  }
1336 
1337  return DAG.getMergeValues(Constants, getCurSDLoc());
1338  }
1339 
1340  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1341  return DAG.getBlockAddress(BA, VT);
1342 
1343  VectorType *VecTy = cast<VectorType>(V->getType());
1344  unsigned NumElements = VecTy->getNumElements();
1345 
1346  // Now that we know the number and type of the elements, get that number of
1347  // elements into the Ops array based on what kind of constant it is.
1349  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1350  for (unsigned i = 0; i != NumElements; ++i)
1351  Ops.push_back(getValue(CV->getOperand(i)));
1352  } else {
1353  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1354  EVT EltVT =
1355  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1356 
1357  SDValue Op;
1358  if (EltVT.isFloatingPoint())
1359  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1360  else
1361  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1362  Ops.assign(NumElements, Op);
1363  }
1364 
1365  // Create a BUILD_VECTOR node.
1366  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1367  }
1368 
1369  // If this is a static alloca, generate it as the frameindex instead of
1370  // computation.
1371  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1373  FuncInfo.StaticAllocaMap.find(AI);
1374  if (SI != FuncInfo.StaticAllocaMap.end())
1375  return DAG.getFrameIndex(SI->second,
1376  TLI.getFrameIndexTy(DAG.getDataLayout()));
1377  }
1378 
1379  // If this is an instruction which fast-isel has deferred, select it now.
1380  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1381  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1382 
1383  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1384  Inst->getType(), getABIRegCopyCC(V));
1385  SDValue Chain = DAG.getEntryNode();
1386  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1387  }
1388 
1389  llvm_unreachable("Can't get register for value!");
1390 }
1391 
1392 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1393  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1394  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1395  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1396  bool IsSEH = isAsynchronousEHPersonality(Pers);
1397  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1398  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1399  if (!IsSEH)
1400  CatchPadMBB->setIsEHScopeEntry();
1401  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1402  if (IsMSVCCXX || IsCoreCLR)
1403  CatchPadMBB->setIsEHFuncletEntry();
1404  // Wasm does not need catchpads anymore
1405  if (!IsWasmCXX)
1406  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1407  getControlRoot()));
1408 }
1409 
1410 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1411  // Update machine-CFG edge.
1412  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1413  FuncInfo.MBB->addSuccessor(TargetMBB);
1414 
1415  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1416  bool IsSEH = isAsynchronousEHPersonality(Pers);
1417  if (IsSEH) {
1418  // If this is not a fall-through branch or optimizations are switched off,
1419  // emit the branch.
1420  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1421  TM.getOptLevel() == CodeGenOpt::None)
1422  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1423  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1424  return;
1425  }
1426 
1427  // Figure out the funclet membership for the catchret's successor.
1428  // This will be used by the FuncletLayout pass to determine how to order the
1429  // BB's.
1430  // A 'catchret' returns to the outer scope's color.
1431  Value *ParentPad = I.getCatchSwitchParentPad();
1432  const BasicBlock *SuccessorColor;
1433  if (isa<ConstantTokenNone>(ParentPad))
1434  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1435  else
1436  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1437  assert(SuccessorColor && "No parent funclet for catchret!");
1438  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1439  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1440 
1441  // Create the terminator node.
1442  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1443  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1444  DAG.getBasicBlock(SuccessorColorMBB));
1445  DAG.setRoot(Ret);
1446 }
1447 
1448 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1449  // Don't emit any special code for the cleanuppad instruction. It just marks
1450  // the start of an EH scope/funclet.
1451  FuncInfo.MBB->setIsEHScopeEntry();
1452  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1453  if (Pers != EHPersonality::Wasm_CXX) {
1454  FuncInfo.MBB->setIsEHFuncletEntry();
1455  FuncInfo.MBB->setIsCleanupFuncletEntry();
1456  }
1457 }
1458 
1459 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1460 /// many places it could ultimately go. In the IR, we have a single unwind
1461 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1462 /// This function skips over imaginary basic blocks that hold catchswitch
1463 /// instructions, and finds all the "real" machine
1464 /// basic block destinations. As those destinations may not be successors of
1465 /// EHPadBB, here we also calculate the edge probability to those destinations.
1466 /// The passed-in Prob is the edge probability to EHPadBB.
1468  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1469  BranchProbability Prob,
1470  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1471  &UnwindDests) {
1472  EHPersonality Personality =
1474  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1475  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1476  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1477  bool IsSEH = isAsynchronousEHPersonality(Personality);
1478 
1479  while (EHPadBB) {
1480  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1481  BasicBlock *NewEHPadBB = nullptr;
1482  if (isa<LandingPadInst>(Pad)) {
1483  // Stop on landingpads. They are not funclets.
1484  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1485  break;
1486  } else if (isa<CleanupPadInst>(Pad)) {
1487  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1488  // personalities.
1489  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1490  UnwindDests.back().first->setIsEHScopeEntry();
1491  if (!IsWasmCXX)
1492  UnwindDests.back().first->setIsEHFuncletEntry();
1493  break;
1494  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1495  // Add the catchpad handlers to the possible destinations.
1496  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1497  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1498  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1499  if (IsMSVCCXX || IsCoreCLR)
1500  UnwindDests.back().first->setIsEHFuncletEntry();
1501  if (!IsSEH)
1502  UnwindDests.back().first->setIsEHScopeEntry();
1503  }
1504  NewEHPadBB = CatchSwitch->getUnwindDest();
1505  } else {
1506  continue;
1507  }
1508 
1509  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1510  if (BPI && NewEHPadBB)
1511  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1512  EHPadBB = NewEHPadBB;
1513  }
1514 }
1515 
1516 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1517  // Update successor info.
1519  auto UnwindDest = I.getUnwindDest();
1520  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1521  BranchProbability UnwindDestProb =
1522  (BPI && UnwindDest)
1523  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1525  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1526  for (auto &UnwindDest : UnwindDests) {
1527  UnwindDest.first->setIsEHPad();
1528  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1529  }
1530  FuncInfo.MBB->normalizeSuccProbs();
1531 
1532  // Create the terminator node.
1533  SDValue Ret =
1534  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1535  DAG.setRoot(Ret);
1536 }
1537 
1538 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1539  report_fatal_error("visitCatchSwitch not yet implemented!");
1540 }
1541 
1542 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1543  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1544  auto &DL = DAG.getDataLayout();
1545  SDValue Chain = getControlRoot();
1547  SmallVector<SDValue, 8> OutVals;
1548 
1549  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1550  // lower
1551  //
1552  // %val = call <ty> @llvm.experimental.deoptimize()
1553  // ret <ty> %val
1554  //
1555  // differently.
1557  LowerDeoptimizingReturn();
1558  return;
1559  }
1560 
1561  if (!FuncInfo.CanLowerReturn) {
1562  unsigned DemoteReg = FuncInfo.DemoteRegister;
1563  const Function *F = I.getParent()->getParent();
1564 
1565  // Emit a store of the return value through the virtual register.
1566  // Leave Outs empty so that LowerReturn won't try to load return
1567  // registers the usual way.
1568  SmallVector<EVT, 1> PtrValueVTs;
1569  ComputeValueVTs(TLI, DL,
1572  PtrValueVTs);
1573 
1574  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1575  DemoteReg, PtrValueVTs[0]);
1576  SDValue RetOp = getValue(I.getOperand(0));
1577 
1580  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1581  unsigned NumValues = ValueVTs.size();
1582 
1583  SmallVector<SDValue, 4> Chains(NumValues);
1584  for (unsigned i = 0; i != NumValues; ++i) {
1585  // An aggregate return value cannot wrap around the address space, so
1586  // offsets to its parts don't wrap either.
1587  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1588  Chains[i] = DAG.getStore(
1589  Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1590  // FIXME: better loc info would be nice.
1592  }
1593 
1594  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1595  MVT::Other, Chains);
1596  } else if (I.getNumOperands() != 0) {
1598  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1599  unsigned NumValues = ValueVTs.size();
1600  if (NumValues) {
1601  SDValue RetOp = getValue(I.getOperand(0));
1602 
1603  const Function *F = I.getParent()->getParent();
1604 
1605  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1606  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1607  Attribute::SExt))
1608  ExtendKind = ISD::SIGN_EXTEND;
1609  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1610  Attribute::ZExt))
1611  ExtendKind = ISD::ZERO_EXTEND;
1612 
1613  LLVMContext &Context = F->getContext();
1614  bool RetInReg = F->getAttributes().hasAttribute(
1615  AttributeList::ReturnIndex, Attribute::InReg);
1616 
1617  for (unsigned j = 0; j != NumValues; ++j) {
1618  EVT VT = ValueVTs[j];
1619 
1620  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1621  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1622 
1623  CallingConv::ID CC = F->getCallingConv();
1624 
1625  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1626  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1627  SmallVector<SDValue, 4> Parts(NumParts);
1628  getCopyToParts(DAG, getCurSDLoc(),
1629  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1630  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1631 
1632  // 'inreg' on function refers to return value
1633  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1634  if (RetInReg)
1635  Flags.setInReg();
1636 
1637  // Propagate extension type if any
1638  if (ExtendKind == ISD::SIGN_EXTEND)
1639  Flags.setSExt();
1640  else if (ExtendKind == ISD::ZERO_EXTEND)
1641  Flags.setZExt();
1642 
1643  for (unsigned i = 0; i < NumParts; ++i) {
1644  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1645  VT, /*isfixed=*/true, 0, 0));
1646  OutVals.push_back(Parts[i]);
1647  }
1648  }
1649  }
1650  }
1651 
1652  // Push in swifterror virtual register as the last element of Outs. This makes
1653  // sure swifterror virtual register will be returned in the swifterror
1654  // physical register.
1655  const Function *F = I.getParent()->getParent();
1656  if (TLI.supportSwiftError() &&
1657  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1658  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1659  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1660  Flags.setSwiftError();
1661  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1662  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1663  true /*isfixed*/, 1 /*origidx*/,
1664  0 /*partOffs*/));
1665  // Create SDNode for the swifterror virtual register.
1666  OutVals.push_back(
1667  DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1668  &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1669  EVT(TLI.getPointerTy(DL))));
1670  }
1671 
1672  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1673  CallingConv::ID CallConv =
1675  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1676  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1677 
1678  // Verify that the target's LowerReturn behaved as expected.
1679  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1680  "LowerReturn didn't return a valid chain!");
1681 
1682  // Update the DAG with the new chain value resulting from return lowering.
1683  DAG.setRoot(Chain);
1684 }
1685 
1686 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1687 /// created for it, emit nodes to copy the value into the virtual
1688 /// registers.
1690  // Skip empty types
1691  if (V->getType()->isEmptyTy())
1692  return;
1693 
1694  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1695  if (VMI != FuncInfo.ValueMap.end()) {
1696  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1697  CopyValueToVirtualRegister(V, VMI->second);
1698  }
1699 }
1700 
1701 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1702 /// the current basic block, add it to ValueMap now so that we'll get a
1703 /// CopyTo/FromReg.
1705  // No need to export constants.
1706  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1707 
1708  // Already exported?
1709  if (FuncInfo.isExportedInst(V)) return;
1710 
1711  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1712  CopyValueToVirtualRegister(V, Reg);
1713 }
1714 
1716  const BasicBlock *FromBB) {
1717  // The operands of the setcc have to be in this block. We don't know
1718  // how to export them from some other block.
1719  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1720  // Can export from current BB.
1721  if (VI->getParent() == FromBB)
1722  return true;
1723 
1724  // Is already exported, noop.
1725  return FuncInfo.isExportedInst(V);
1726  }
1727 
1728  // If this is an argument, we can export it if the BB is the entry block or
1729  // if it is already exported.
1730  if (isa<Argument>(V)) {
1731  if (FromBB == &FromBB->getParent()->getEntryBlock())
1732  return true;
1733 
1734  // Otherwise, can only export this if it is already exported.
1735  return FuncInfo.isExportedInst(V);
1736  }
1737 
1738  // Otherwise, constants can always be exported.
1739  return true;
1740 }
1741 
1742 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1744 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1745  const MachineBasicBlock *Dst) const {
1746  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1747  const BasicBlock *SrcBB = Src->getBasicBlock();
1748  const BasicBlock *DstBB = Dst->getBasicBlock();
1749  if (!BPI) {
1750  // If BPI is not available, set the default probability as 1 / N, where N is
1751  // the number of successors.
1752  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1753  return BranchProbability(1, SuccSize);
1754  }
1755  return BPI->getEdgeProbability(SrcBB, DstBB);
1756 }
1757 
1758 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1759  MachineBasicBlock *Dst,
1760  BranchProbability Prob) {
1761  if (!FuncInfo.BPI)
1762  Src->addSuccessorWithoutProb(Dst);
1763  else {
1764  if (Prob.isUnknown())
1765  Prob = getEdgeProbability(Src, Dst);
1766  Src->addSuccessor(Dst, Prob);
1767  }
1768 }
1769 
1770 static bool InBlock(const Value *V, const BasicBlock *BB) {
1771  if (const Instruction *I = dyn_cast<Instruction>(V))
1772  return I->getParent() == BB;
1773  return true;
1774 }
1775 
1776 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1777 /// This function emits a branch and is used at the leaves of an OR or an
1778 /// AND operator tree.
1779 void
1781  MachineBasicBlock *TBB,
1782  MachineBasicBlock *FBB,
1783  MachineBasicBlock *CurBB,
1784  MachineBasicBlock *SwitchBB,
1785  BranchProbability TProb,
1786  BranchProbability FProb,
1787  bool InvertCond) {
1788  const BasicBlock *BB = CurBB->getBasicBlock();
1789 
1790  // If the leaf of the tree is a comparison, merge the condition into
1791  // the caseblock.
1792  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1793  // The operands of the cmp have to be in this block. We don't know
1794  // how to export them from some other block. If this is the first block
1795  // of the sequence, no exporting is needed.
1796  if (CurBB == SwitchBB ||
1797  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1798  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1799  ISD::CondCode Condition;
1800  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1801  ICmpInst::Predicate Pred =
1802  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1803  Condition = getICmpCondCode(Pred);
1804  } else {
1805  const FCmpInst *FC = cast<FCmpInst>(Cond);
1806  FCmpInst::Predicate Pred =
1807  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1808  Condition = getFCmpCondCode(Pred);
1809  if (TM.Options.NoNaNsFPMath)
1810  Condition = getFCmpCodeWithoutNaN(Condition);
1811  }
1812 
1813  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1814  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1815  SwitchCases.push_back(CB);
1816  return;
1817  }
1818  }
1819 
1820  // Create a CaseBlock record representing this branch.
1821  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1822  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1823  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1824  SwitchCases.push_back(CB);
1825 }
1826 
1827 /// FindMergedConditions - If Cond is an expression like
1829  MachineBasicBlock *TBB,
1830  MachineBasicBlock *FBB,
1831  MachineBasicBlock *CurBB,
1832  MachineBasicBlock *SwitchBB,
1834  BranchProbability TProb,
1835  BranchProbability FProb,
1836  bool InvertCond) {
1837  // Skip over not part of the tree and remember to invert op and operands at
1838  // next level.
1839  if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1840  const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1841  if (InBlock(CondOp, CurBB->getBasicBlock())) {
1842  FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1843  !InvertCond);
1844  return;
1845  }
1846  }
1847 
1848  const Instruction *BOp = dyn_cast<Instruction>(Cond);
1849  // Compute the effective opcode for Cond, taking into account whether it needs
1850  // to be inverted, e.g.
1851  // and (not (or A, B)), C
1852  // gets lowered as
1853  // and (and (not A, not B), C)
1854  unsigned BOpc = 0;
1855  if (BOp) {
1856  BOpc = BOp->getOpcode();
1857  if (InvertCond) {
1858  if (BOpc == Instruction::And)
1859  BOpc = Instruction::Or;
1860  else if (BOpc == Instruction::Or)
1861  BOpc = Instruction::And;
1862  }
1863  }
1864 
1865  // If this node is not part of the or/and tree, emit it as a branch.
1866  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1867  BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1868  BOp->getParent() != CurBB->getBasicBlock() ||
1869  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1870  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1871  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1872  TProb, FProb, InvertCond);
1873  return;
1874  }
1875 
1876  // Create TmpBB after CurBB.
1877  MachineFunction::iterator BBI(CurBB);
1878  MachineFunction &MF = DAG.getMachineFunction();
1880  CurBB->getParent()->insert(++BBI, TmpBB);
1881 
1882  if (Opc == Instruction::Or) {
1883  // Codegen X | Y as:
1884  // BB1:
1885  // jmp_if_X TBB
1886  // jmp TmpBB
1887  // TmpBB:
1888  // jmp_if_Y TBB
1889  // jmp FBB
1890  //
1891 
1892  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1893  // The requirement is that
1894  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1895  // = TrueProb for original BB.
1896  // Assuming the original probabilities are A and B, one choice is to set
1897  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1898  // A/(1+B) and 2B/(1+B). This choice assumes that
1899  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1900  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1901  // TmpBB, but the math is more complicated.
1902 
1903  auto NewTrueProb = TProb / 2;
1904  auto NewFalseProb = TProb / 2 + FProb;
1905  // Emit the LHS condition.
1906  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1907  NewTrueProb, NewFalseProb, InvertCond);
1908 
1909  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1910  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1911  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1912  // Emit the RHS condition into TmpBB.
1913  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1914  Probs[0], Probs[1], InvertCond);
1915  } else {
1916  assert(Opc == Instruction::And && "Unknown merge op!");
1917  // Codegen X & Y as:
1918  // BB1:
1919  // jmp_if_X TmpBB
1920  // jmp FBB
1921  // TmpBB:
1922  // jmp_if_Y TBB
1923  // jmp FBB
1924  //
1925  // This requires creation of TmpBB after CurBB.
1926 
1927  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1928  // The requirement is that
1929  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1930  // = FalseProb for original BB.
1931  // Assuming the original probabilities are A and B, one choice is to set
1932  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1933  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1934  // TrueProb for BB1 * FalseProb for TmpBB.
1935 
1936  auto NewTrueProb = TProb + FProb / 2;
1937  auto NewFalseProb = FProb / 2;
1938  // Emit the LHS condition.
1939  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1940  NewTrueProb, NewFalseProb, InvertCond);
1941 
1942  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1943  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1944  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1945  // Emit the RHS condition into TmpBB.
1946  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1947  Probs[0], Probs[1], InvertCond);
1948  }
1949 }
1950 
1951 /// If the set of cases should be emitted as a series of branches, return true.
1952 /// If we should emit this as a bunch of and/or'd together conditions, return
1953 /// false.
1954 bool
1955 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1956  if (Cases.size() != 2) return true;
1957 
1958  // If this is two comparisons of the same values or'd or and'd together, they
1959  // will get folded into a single comparison, so don't emit two blocks.
1960  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1961  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1962  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1963  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1964  return false;
1965  }
1966 
1967  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1968  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1969  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1970  Cases[0].CC == Cases[1].CC &&
1971  isa<Constant>(Cases[0].CmpRHS) &&
1972  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1973  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1974  return false;
1975  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1976  return false;
1977  }
1978 
1979  return true;
1980 }
1981 
1982 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1983  MachineBasicBlock *BrMBB = FuncInfo.MBB;
1984 
1985  // Update machine-CFG edges.
1986  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1987 
1988  if (I.isUnconditional()) {
1989  // Update machine-CFG edges.
1990  BrMBB->addSuccessor(Succ0MBB);
1991 
1992  // If this is not a fall-through branch or optimizations are switched off,
1993  // emit the branch.
1994  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1995  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1996  MVT::Other, getControlRoot(),
1997  DAG.getBasicBlock(Succ0MBB)));
1998 
1999  return;
2000  }
2001 
2002  // If this condition is one of the special cases we handle, do special stuff
2003  // now.
2004  const Value *CondVal = I.getCondition();
2005  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2006 
2007  // If this is a series of conditions that are or'd or and'd together, emit
2008  // this as a sequence of branches instead of setcc's with and/or operations.
2009  // As long as jumps are not expensive, this should improve performance.
2010  // For example, instead of something like:
2011  // cmp A, B
2012  // C = seteq
2013  // cmp D, E
2014  // F = setle
2015  // or C, F
2016  // jnz foo
2017  // Emit:
2018  // cmp A, B
2019  // je foo
2020  // cmp D, E
2021  // jle foo
2022  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2023  Instruction::BinaryOps Opcode = BOp->getOpcode();
2024  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2026  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2027  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2028  Opcode,
2029  getEdgeProbability(BrMBB, Succ0MBB),
2030  getEdgeProbability(BrMBB, Succ1MBB),
2031  /*InvertCond=*/false);
2032  // If the compares in later blocks need to use values not currently
2033  // exported from this block, export them now. This block should always
2034  // be the first entry.
2035  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2036 
2037  // Allow some cases to be rejected.
2038  if (ShouldEmitAsBranches(SwitchCases)) {
2039  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2040  ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2041  ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2042  }
2043 
2044  // Emit the branch for this block.
2045  visitSwitchCase(SwitchCases[0], BrMBB);
2046  SwitchCases.erase(SwitchCases.begin());
2047  return;
2048  }
2049 
2050  // Okay, we decided not to do this, remove any inserted MBB's and clear
2051  // SwitchCases.
2052  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2053  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2054 
2055  SwitchCases.clear();
2056  }
2057  }
2058 
2059  // Create a CaseBlock record representing this branch.
2060  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2061  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2062 
2063  // Use visitSwitchCase to actually insert the fast branch sequence for this
2064  // cond branch.
2065  visitSwitchCase(CB, BrMBB);
2066 }
2067 
2068 /// visitSwitchCase - Emits the necessary code to represent a single node in
2069 /// the binary search tree resulting from lowering a switch instruction.
2071  MachineBasicBlock *SwitchBB) {
2072  SDValue Cond;
2073  SDValue CondLHS = getValue(CB.CmpLHS);
2074  SDLoc dl = CB.DL;
2075 
2076  // Build the setcc now.
2077  if (!CB.CmpMHS) {
2078  // Fold "(X == true)" to X and "(X == false)" to !X to
2079  // handle common cases produced by branch lowering.
2080  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2081  CB.CC == ISD::SETEQ)
2082  Cond = CondLHS;
2083  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2084  CB.CC == ISD::SETEQ) {
2085  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2086  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2087  } else
2088  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2089  } else {
2090  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2091 
2092  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2093  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2094 
2095  SDValue CmpOp = getValue(CB.CmpMHS);
2096  EVT VT = CmpOp.getValueType();
2097 
2098  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2099  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2100  ISD::SETLE);
2101  } else {
2102  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2103  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2104  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2105  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2106  }
2107  }
2108 
2109  // Update successor info
2110  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2111  // TrueBB and FalseBB are always different unless the incoming IR is
2112  // degenerate. This only happens when running llc on weird IR.
2113  if (CB.TrueBB != CB.FalseBB)
2114  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2115  SwitchBB->normalizeSuccProbs();
2116 
2117  // If the lhs block is the next block, invert the condition so that we can
2118  // fall through to the lhs instead of the rhs block.
2119  if (CB.TrueBB == NextBlock(SwitchBB)) {
2120  std::swap(CB.TrueBB, CB.FalseBB);
2121  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2122  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2123  }
2124 
2125  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2126  MVT::Other, getControlRoot(), Cond,
2127  DAG.getBasicBlock(CB.TrueBB));
2128 
2129  // Insert the false branch. Do this even if it's a fall through branch,
2130  // this makes it easier to do DAG optimizations which require inverting
2131  // the branch condition.
2132  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2133  DAG.getBasicBlock(CB.FalseBB));
2134 
2135  DAG.setRoot(BrCond);
2136 }
2137 
2138 /// visitJumpTable - Emit JumpTable node in the current MBB
2140  // Emit the code for the jump table
2141  assert(JT.Reg != -1U && "Should lower JT Header first!");
2143  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2144  JT.Reg, PTy);
2145  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2146  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2147  MVT::Other, Index.getValue(1),
2148  Table, Index);
2149  DAG.setRoot(BrJumpTable);
2150 }
2151 
2152 /// visitJumpTableHeader - This function emits necessary code to produce index
2153 /// in the JumpTable from switch case.
2155  JumpTableHeader &JTH,
2156  MachineBasicBlock *SwitchBB) {
2157  SDLoc dl = getCurSDLoc();
2158 
2159  // Subtract the lowest switch case value from the value being switched on and
2160  // conditional branch to default mbb if the result is greater than the
2161  // difference between smallest and largest cases.
2162  SDValue SwitchOp = getValue(JTH.SValue);
2163  EVT VT = SwitchOp.getValueType();
2164  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2165  DAG.getConstant(JTH.First, dl, VT));
2166 
2167  // The SDNode we just created, which holds the value being switched on minus
2168  // the smallest case value, needs to be copied to a virtual register so it
2169  // can be used as an index into the jump table in a subsequent basic block.
2170  // This value may be smaller or larger than the target's pointer type, and
2171  // therefore require extension or truncating.
2172  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2173  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2174 
2175  unsigned JumpTableReg =
2176  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2177  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2178  JumpTableReg, SwitchOp);
2179  JT.Reg = JumpTableReg;
2180 
2181  // Emit the range check for the jump table, and branch to the default block
2182  // for the switch statement if the value being switched on exceeds the largest
2183  // case in the switch.
2184  SDValue CMP = DAG.getSetCC(
2185  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2186  Sub.getValueType()),
2187  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2188 
2189  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2190  MVT::Other, CopyTo, CMP,
2191  DAG.getBasicBlock(JT.Default));
2192 
2193  // Avoid emitting unnecessary branches to the next block.
2194  if (JT.MBB != NextBlock(SwitchBB))
2195  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2196  DAG.getBasicBlock(JT.MBB));
2197 
2198  DAG.setRoot(BrCond);
2199 }
2200 
2201 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2202 /// variable if there exists one.
2204  SDValue &Chain) {
2205  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2206  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2207  MachineFunction &MF = DAG.getMachineFunction();
2208  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2209  MachineSDNode *Node =
2210  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2211  if (Global) {
2212  MachinePointerInfo MPInfo(Global);
2216  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2217  DAG.setNodeMemRefs(Node, {MemRef});
2218  }
2219  return SDValue(Node, 0);
2220 }
2221 
2222 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2223 /// tail spliced into a stack protector check success bb.
2224 ///
2225 /// For a high level explanation of how this fits into the stack protector
2226 /// generation see the comment on the declaration of class
2227 /// StackProtectorDescriptor.
2228 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2229  MachineBasicBlock *ParentBB) {
2230 
2231  // First create the loads to the guard/stack slot for the comparison.
2232  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2233  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2234 
2235  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2236  int FI = MFI.getStackProtectorIndex();
2237 
2238  SDValue Guard;
2239  SDLoc dl = getCurSDLoc();
2240  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2241  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2242  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2243 
2244  // Generate code to load the content of the guard slot.
2245  SDValue GuardVal = DAG.getLoad(
2246  PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2249 
2250  if (TLI.useStackGuardXorFP())
2251  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2252 
2253  // Retrieve guard check function, nullptr if instrumentation is inlined.
2254  if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2255  // The target provides a guard check function to validate the guard value.
2256  // Generate a call to that function with the content of the guard slot as
2257  // argument.
2258  auto *Fn = cast<Function>(GuardCheck);
2259  FunctionType *FnTy = Fn->getFunctionType();
2260  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2261 
2264  Entry.Node = GuardVal;
2265  Entry.Ty = FnTy->getParamType(0);
2266  if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2267  Entry.IsInReg = true;
2268  Args.push_back(Entry);
2269 
2271  CLI.setDebugLoc(getCurSDLoc())
2272  .setChain(DAG.getEntryNode())
2273  .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2274  getValue(GuardCheck), std::move(Args));
2275 
2276  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2277  DAG.setRoot(Result.second);
2278  return;
2279  }
2280 
2281  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2282  // Otherwise, emit a volatile load to retrieve the stack guard value.
2283  SDValue Chain = DAG.getEntryNode();
2284  if (TLI.useLoadStackGuardNode()) {
2285  Guard = getLoadStackGuard(DAG, dl, Chain);
2286  } else {
2287  const Value *IRGuard = TLI.getSDagStackGuard(M);
2288  SDValue GuardPtr = getValue(IRGuard);
2289 
2290  Guard =
2291  DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2293  }
2294 
2295  // Perform the comparison via a subtract/getsetcc.
2296  EVT VT = Guard.getValueType();
2297  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2298 
2299  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2300  *DAG.getContext(),
2301  Sub.getValueType()),
2302  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2303 
2304  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2305  // branch to failure MBB.
2306  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2307  MVT::Other, GuardVal.getOperand(0),
2308  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2309  // Otherwise branch to success MBB.
2310  SDValue Br = DAG.getNode(ISD::BR, dl,
2311  MVT::Other, BrCond,
2312  DAG.getBasicBlock(SPD.getSuccessMBB()));
2313 
2314  DAG.setRoot(Br);
2315 }
2316 
2317 /// Codegen the failure basic block for a stack protector check.
2318 ///
2319 /// A failure stack protector machine basic block consists simply of a call to
2320 /// __stack_chk_fail().
2321 ///
2322 /// For a high level explanation of how this fits into the stack protector
2323 /// generation see the comment on the declaration of class
2324 /// StackProtectorDescriptor.
2325 void
2326 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2327  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2328  SDValue Chain =
2329  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2330  None, false, getCurSDLoc(), false, false).second;
2331  DAG.setRoot(Chain);
2332 }
2333 
2334 /// visitBitTestHeader - This function emits necessary code to produce value
2335 /// suitable for "bit tests"
2337  MachineBasicBlock *SwitchBB) {
2338  SDLoc dl = getCurSDLoc();
2339 
2340  // Subtract the minimum value
2341  SDValue SwitchOp = getValue(B.SValue);
2342  EVT VT = SwitchOp.getValueType();
2343  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2344  DAG.getConstant(B.First, dl, VT));
2345 
2346  // Check range
2347  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2348  SDValue RangeCmp = DAG.getSetCC(
2349  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2350  Sub.getValueType()),
2351  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2352 
2353  // Determine the type of the test operands.
2354  bool UsePtrType = false;
2355  if (!TLI.isTypeLegal(VT))
2356  UsePtrType = true;
2357  else {
2358  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2359  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2360  // Switch table case range are encoded into series of masks.
2361  // Just use pointer type, it's guaranteed to fit.
2362  UsePtrType = true;
2363  break;
2364  }
2365  }
2366  if (UsePtrType) {
2367  VT = TLI.getPointerTy(DAG.getDataLayout());
2368  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2369  }
2370 
2371  B.RegVT = VT.getSimpleVT();
2372  B.Reg = FuncInfo.CreateReg(B.RegVT);
2373  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2374 
2375  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2376 
2377  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2378  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2379  SwitchBB->normalizeSuccProbs();
2380 
2381  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2382  MVT::Other, CopyTo, RangeCmp,
2383  DAG.getBasicBlock(B.Default));
2384 
2385  // Avoid emitting unnecessary branches to the next block.
2386  if (MBB != NextBlock(SwitchBB))
2387  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2388  DAG.getBasicBlock(MBB));
2389 
2390  DAG.setRoot(BrRange);
2391 }
2392 
2393 /// visitBitTestCase - this function produces one "bit test"
2395  MachineBasicBlock* NextMBB,
2396  BranchProbability BranchProbToNext,
2397  unsigned Reg,
2398  BitTestCase &B,
2399  MachineBasicBlock *SwitchBB) {
2400  SDLoc dl = getCurSDLoc();
2401  MVT VT = BB.RegVT;
2402  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2403  SDValue Cmp;
2404  unsigned PopCount = countPopulation(B.Mask);
2405  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2406  if (PopCount == 1) {
2407  // Testing for a single bit; just compare the shift count with what it
2408  // would need to be to shift a 1 bit in that position.
2409  Cmp = DAG.getSetCC(
2410  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2411  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2412  ISD::SETEQ);
2413  } else if (PopCount == BB.Range) {
2414  // There is only one zero bit in the range, test for it directly.
2415  Cmp = DAG.getSetCC(
2416  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2417  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2418  ISD::SETNE);
2419  } else {
2420  // Make desired shift
2421  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2422  DAG.getConstant(1, dl, VT), ShiftOp);
2423 
2424  // Emit bit tests and jumps
2425  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2426  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2427  Cmp = DAG.getSetCC(
2428  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2429  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2430  }
2431 
2432  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2433  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2434  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2435  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2436  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2437  // one as they are relative probabilities (and thus work more like weights),
2438  // and hence we need to normalize them to let the sum of them become one.
2439  SwitchBB->normalizeSuccProbs();
2440 
2441  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2442  MVT::Other, getControlRoot(),
2443  Cmp, DAG.getBasicBlock(B.TargetBB));
2444 
2445  // Avoid emitting unnecessary branches to the next block.
2446  if (NextMBB != NextBlock(SwitchBB))
2447  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2448  DAG.getBasicBlock(NextMBB));
2449 
2450  DAG.setRoot(BrAnd);
2451 }
2452 
2453 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2454  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2455 
2456  // Retrieve successors. Look through artificial IR level blocks like
2457  // catchswitch for successors.
2458  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2459  const BasicBlock *EHPadBB = I.getSuccessor(1);
2460 
2461  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2462  // have to do anything here to lower funclet bundles.
2464  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2465  "Cannot lower invokes with arbitrary operand bundles yet!");
2466 
2467  const Value *Callee(I.getCalledValue());
2468  const Function *Fn = dyn_cast<Function>(Callee);
2469  if (isa<InlineAsm>(Callee))
2470  visitInlineAsm(&I);
2471  else if (Fn && Fn->isIntrinsic()) {
2472  switch (Fn->getIntrinsicID()) {
2473  default:
2474  llvm_unreachable("Cannot invoke this intrinsic");
2475  case Intrinsic::donothing:
2476  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2477  break;
2478  case Intrinsic::experimental_patchpoint_void:
2479  case Intrinsic::experimental_patchpoint_i64:
2480  visitPatchpoint(&I, EHPadBB);
2481  break;
2482  case Intrinsic::experimental_gc_statepoint:
2483  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2484  break;
2485  }
2487  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2488  // Eventually we will support lowering the @llvm.experimental.deoptimize
2489  // intrinsic, and right now there are no plans to support other intrinsics
2490  // with deopt state.
2491  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2492  } else {
2493  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2494  }
2495 
2496  // If the value of the invoke is used outside of its defining block, make it
2497  // available as a virtual register.
2498  // We already took care of the exported value for the statepoint instruction
2499  // during call to the LowerStatepoint.
2500  if (!isStatepoint(I)) {
2501  CopyToExportRegsIfNeeded(&I);
2502  }
2503 
2505  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2506  BranchProbability EHPadBBProb =
2507  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2509  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2510 
2511  // Update successor info.
2512  addSuccessorWithProb(InvokeMBB, Return);
2513  for (auto &UnwindDest : UnwindDests) {
2514  UnwindDest.first->setIsEHPad();
2515  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2516  }
2517  InvokeMBB->normalizeSuccProbs();
2518 
2519  // Drop into normal successor.
2520  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2521  MVT::Other, getControlRoot(),
2522  DAG.getBasicBlock(Return)));
2523 }
2524 
2525 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2526  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2527 }
2528 
2529 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2530  assert(FuncInfo.MBB->isEHPad() &&
2531  "Call to landingpad not in landing pad!");
2532 
2533  MachineBasicBlock *MBB = FuncInfo.MBB;
2534  addLandingPadInfo(LP, *MBB);
2535 
2536  // If there aren't registers to copy the values into (e.g., during SjLj
2537  // exceptions), then don't bother to create these DAG nodes.
2538  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2539  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2540  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2541  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2542  return;
2543 
2544  // If landingpad's return type is token type, we don't create DAG nodes
2545  // for its exception pointer and selector value. The extraction of exception
2546  // pointer or selector value from token type landingpads is not currently
2547  // supported.
2548  if (LP.getType()->isTokenTy())
2549  return;
2550 
2552  SDLoc dl = getCurSDLoc();
2553  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2554  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2555 
2556  // Get the two live-in registers as SDValues. The physregs have already been
2557  // copied into virtual registers.
2558  SDValue Ops[2];
2559  if (FuncInfo.ExceptionPointerVirtReg) {
2560  Ops[0] = DAG.getZExtOrTrunc(
2561  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2562  FuncInfo.ExceptionPointerVirtReg,
2563  TLI.getPointerTy(DAG.getDataLayout())),
2564  dl, ValueVTs[0]);
2565  } else {
2566  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2567  }
2568  Ops[1] = DAG.getZExtOrTrunc(
2569  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2570  FuncInfo.ExceptionSelectorVirtReg,
2571  TLI.getPointerTy(DAG.getDataLayout())),
2572  dl, ValueVTs[1]);
2573 
2574  // Merge into one.
2575  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2576  DAG.getVTList(ValueVTs), Ops);
2577  setValue(&LP, Res);
2578 }
2579 
2580 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2581 #ifndef NDEBUG
2582  for (const CaseCluster &CC : Clusters)
2583  assert(CC.Low == CC.High && "Input clusters must be single-case");
2584 #endif
2585 
2586  llvm::sort(Clusters.begin(), Clusters.end(),
2587  [](const CaseCluster &a, const CaseCluster &b) {
2588  return a.Low->getValue().slt(b.Low->getValue());
2589  });
2590 
2591  // Merge adjacent clusters with the same destination.
2592  const unsigned N = Clusters.size();
2593  unsigned DstIndex = 0;
2594  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2595  CaseCluster &CC = Clusters[SrcIndex];
2596  const ConstantInt *CaseVal = CC.Low;
2597  MachineBasicBlock *Succ = CC.MBB;
2598 
2599  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2600  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2601  // If this case has the same successor and is a neighbour, merge it into
2602  // the previous cluster.
2603  Clusters[DstIndex - 1].High = CaseVal;
2604  Clusters[DstIndex - 1].Prob += CC.Prob;
2605  } else {
2606  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2607  sizeof(Clusters[SrcIndex]));
2608  }
2609  }
2610  Clusters.resize(DstIndex);
2611 }
2612 
2614  MachineBasicBlock *Last) {
2615  // Update JTCases.
2616  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2617  if (JTCases[i].first.HeaderBB == First)
2618  JTCases[i].first.HeaderBB = Last;
2619 
2620  // Update BitTestCases.
2621  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2622  if (BitTestCases[i].Parent == First)
2623  BitTestCases[i].Parent = Last;
2624 }
2625 
2626 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2627  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2628 
2629  // Update machine-CFG edges with unique successors.
2631  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2632  BasicBlock *BB = I.getSuccessor(i);
2633  bool Inserted = Done.insert(BB).second;
2634  if (!Inserted)
2635  continue;
2636 
2637  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2638  addSuccessorWithProb(IndirectBrMBB, Succ);
2639  }
2640  IndirectBrMBB->normalizeSuccProbs();
2641 
2642  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2643  MVT::Other, getControlRoot(),
2644  getValue(I.getAddress())));
2645 }
2646 
2647 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2648  if (!DAG.getTarget().Options.TrapUnreachable)
2649  return;
2650 
2651  // We may be able to ignore unreachable behind a noreturn call.
2653  const BasicBlock &BB = *I.getParent();
2654  if (&I != &BB.front()) {
2656  std::prev(BasicBlock::const_iterator(&I));
2657  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2658  if (Call->doesNotReturn())
2659  return;
2660  }
2661  }
2662  }
2663 
2664  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2665 }
2666 
2667 void SelectionDAGBuilder::visitFSub(const User &I) {
2668  // -0.0 - X --> fneg
2669  Type *Ty = I.getType();
2670  if (isa<Constant>(I.getOperand(0)) &&
2672  SDValue Op2 = getValue(I.getOperand(1));
2673  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2674  Op2.getValueType(), Op2));
2675  return;
2676  }
2677 
2678  visitBinary(I, ISD::FSUB);
2679 }
2680 
2681 /// Checks if the given instruction performs a vector reduction, in which case
2682 /// we have the freedom to alter the elements in the result as long as the
2683 /// reduction of them stays unchanged.
2684 static bool isVectorReductionOp(const User *I) {
2685  const Instruction *Inst = dyn_cast<Instruction>(I);
2686  if (!Inst || !Inst->getType()->isVectorTy())
2687  return false;
2688 
2689  auto OpCode = Inst->getOpcode();
2690  switch (OpCode) {
2691  case Instruction::Add:
2692  case Instruction::Mul:
2693  case Instruction::And:
2694  case Instruction::Or:
2695  case Instruction::Xor:
2696  break;
2697  case Instruction::FAdd:
2698  case Instruction::FMul:
2699  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2700  if (FPOp->getFastMathFlags().isFast())
2701  break;
2703  default:
2704  return false;
2705  }
2706 
2707  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2708  // Ensure the reduction size is a power of 2.
2709  if (!isPowerOf2_32(ElemNum))
2710  return false;
2711 
2712  unsigned ElemNumToReduce = ElemNum;
2713 
2714  // Do DFS search on the def-use chain from the given instruction. We only
2715  // allow four kinds of operations during the search until we reach the
2716  // instruction that extracts the first element from the vector:
2717  //
2718  // 1. The reduction operation of the same opcode as the given instruction.
2719  //
2720  // 2. PHI node.
2721  //
2722  // 3. ShuffleVector instruction together with a reduction operation that
2723  // does a partial reduction.
2724  //
2725  // 4. ExtractElement that extracts the first element from the vector, and we
2726  // stop searching the def-use chain here.
2727  //
2728  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2729  // from 1-3 to the stack to continue the DFS. The given instruction is not
2730  // a reduction operation if we meet any other instructions other than those
2731  // listed above.
2732 
2733  SmallVector<const User *, 16> UsersToVisit{Inst};
2735  bool ReduxExtracted = false;
2736 
2737  while (!UsersToVisit.empty()) {
2738  auto User = UsersToVisit.back();
2739  UsersToVisit.pop_back();
2740  if (!Visited.insert(User).second)
2741  continue;
2742 
2743  for (const auto &U : User->users()) {
2744  auto Inst = dyn_cast<Instruction>(U);
2745  if (!Inst)
2746  return false;
2747 
2748  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2749  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2750  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2751  return false;
2752  UsersToVisit.push_back(U);
2753  } else if (const ShuffleVectorInst *ShufInst =
2754  dyn_cast<ShuffleVectorInst>(U)) {
2755  // Detect the following pattern: A ShuffleVector instruction together
2756  // with a reduction that do partial reduction on the first and second
2757  // ElemNumToReduce / 2 elements, and store the result in
2758  // ElemNumToReduce / 2 elements in another vector.
2759 
2760  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2761  if (ResultElements < ElemNum)
2762  return false;
2763 
2764  if (ElemNumToReduce == 1)
2765  return false;
2766  if (!isa<UndefValue>(U->getOperand(1)))
2767  return false;
2768  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2769  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2770  return false;
2771  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2772  if (ShufInst->getMaskValue(i) != -1)
2773  return false;
2774 
2775  // There is only one user of this ShuffleVector instruction, which
2776  // must be a reduction operation.
2777  if (!U->hasOneUse())
2778  return false;
2779 
2780  auto U2 = dyn_cast<Instruction>(*U->user_begin());
2781  if (!U2 || U2->getOpcode() != OpCode)
2782  return false;
2783 
2784  // Check operands of the reduction operation.
2785  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2786  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2787  UsersToVisit.push_back(U2);
2788  ElemNumToReduce /= 2;
2789  } else
2790  return false;
2791  } else if (isa<ExtractElementInst>(U)) {
2792  // At this moment we should have reduced all elements in the vector.
2793  if (ElemNumToReduce != 1)
2794  return false;
2795 
2796  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2797  if (!Val || !Val->isZero())
2798  return false;
2799 
2800  ReduxExtracted = true;
2801  } else
2802  return false;
2803  }
2804  }
2805  return ReduxExtracted;
2806 }
2807 
2808 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2809  SDNodeFlags Flags;
2810  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2811  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2812  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2813  }
2814  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2815  Flags.setExact(ExactOp->isExact());
2816  }
2817  if (isVectorReductionOp(&I)) {
2818  Flags.setVectorReduction(true);
2819  LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2820  }
2821 
2822  SDValue Op1 = getValue(I.getOperand(0));
2823  SDValue Op2 = getValue(I.getOperand(1));
2824  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2825  Op1, Op2, Flags);
2826  setValue(&I, BinNodeValue);
2827 }
2828 
2829 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2830  SDValue Op1 = getValue(I.getOperand(0));
2831  SDValue Op2 = getValue(I.getOperand(1));
2832 
2833  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2834  Op2.getValueType(), DAG.getDataLayout());
2835 
2836  // Coerce the shift amount to the right type if we can.
2837  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2838  unsigned ShiftSize = ShiftTy.getSizeInBits();
2839  unsigned Op2Size = Op2.getValueSizeInBits();
2840  SDLoc DL = getCurSDLoc();
2841 
2842  // If the operand is smaller than the shift count type, promote it.
2843  if (ShiftSize > Op2Size)
2844  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2845 
2846  // If the operand is larger than the shift count type but the shift
2847  // count type has enough bits to represent any shift value, truncate
2848  // it now. This is a common case and it exposes the truncate to
2849  // optimization early.
2850  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2851  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2852  // Otherwise we'll need to temporarily settle for some other convenient
2853  // type. Type legalization will make adjustments once the shiftee is split.
2854  else
2855  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2856  }
2857 
2858  bool nuw = false;
2859  bool nsw = false;
2860  bool exact = false;
2861 
2862  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2863 
2864  if (const OverflowingBinaryOperator *OFBinOp =
2865  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2866  nuw = OFBinOp->hasNoUnsignedWrap();
2867  nsw = OFBinOp->hasNoSignedWrap();
2868  }
2869  if (const PossiblyExactOperator *ExactOp =
2870  dyn_cast<const PossiblyExactOperator>(&I))
2871  exact = ExactOp->isExact();
2872  }
2873  SDNodeFlags Flags;
2874  Flags.setExact(exact);
2875  Flags.setNoSignedWrap(nsw);
2876  Flags.setNoUnsignedWrap(nuw);
2877  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2878  Flags);
2879  setValue(&I, Res);
2880 }
2881 
2882 void SelectionDAGBuilder::visitSDiv(const User &I) {
2883  SDValue Op1 = getValue(I.getOperand(0));
2884  SDValue Op2 = getValue(I.getOperand(1));
2885 
2886  SDNodeFlags Flags;
2887  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2888  cast<PossiblyExactOperator>(&I)->isExact());
2889  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2890  Op2, Flags));
2891 }
2892 
2893 void SelectionDAGBuilder::visitICmp(const User &I) {
2895  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2896  predicate = IC->getPredicate();
2897  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2898  predicate = ICmpInst::Predicate(IC->getPredicate());
2899  SDValue Op1 = getValue(I.getOperand(0));
2900  SDValue Op2 = getValue(I.getOperand(1));
2901  ISD::CondCode Opcode = getICmpCondCode(predicate);
2902 
2903  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2904  I.getType());
2905  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2906 }
2907 
2908 void SelectionDAGBuilder::visitFCmp(const User &I) {
2910  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2911  predicate = FC->getPredicate();
2912  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2913  predicate = FCmpInst::Predicate(FC->getPredicate());
2914  SDValue Op1 = getValue(I.getOperand(0));
2915  SDValue Op2 = getValue(I.getOperand(1));
2916 
2917  ISD::CondCode Condition = getFCmpCondCode(predicate);
2918  auto *FPMO = dyn_cast<FPMathOperator>(&I);
2919  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2920  Condition = getFCmpCodeWithoutNaN(Condition);
2921 
2922  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2923  I.getType());
2924  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2925 }
2926 
2927 // Check if the condition of the select has one use or two users that are both
2928 // selects with the same condition.
2929 static bool hasOnlySelectUsers(const Value *Cond) {
2930  return llvm::all_of(Cond->users(), [](const Value *V) {
2931  return isa<SelectInst>(V);
2932  });
2933 }
2934 
2935 void SelectionDAGBuilder::visitSelect(const User &I) {
2938  ValueVTs);
2939  unsigned NumValues = ValueVTs.size();
2940  if (NumValues == 0) return;
2941 
2942  SmallVector<SDValue, 4> Values(NumValues);
2943  SDValue Cond = getValue(I.getOperand(0));
2944  SDValue LHSVal = getValue(I.getOperand(1));
2945  SDValue RHSVal = getValue(I.getOperand(2));
2946  auto BaseOps = {Cond};
2947  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2949 
2950  // Min/max matching is only viable if all output VTs are the same.
2951  if (is_splat(ValueVTs)) {
2952  EVT VT = ValueVTs[0];
2953  LLVMContext &Ctx = *DAG.getContext();
2954  auto &TLI = DAG.getTargetLoweringInfo();
2955 
2956  // We care about the legality of the operation after it has been type
2957  // legalized.
2958  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2959  VT != TLI.getTypeToTransformTo(Ctx, VT))
2960  VT = TLI.getTypeToTransformTo(Ctx, VT);
2961 
2962  // If the vselect is legal, assume we want to leave this as a vector setcc +
2963  // vselect. Otherwise, if this is going to be scalarized, we want to see if
2964  // min/max is legal on the scalar type.
2965  bool UseScalarMinMax = VT.isVector() &&
2966  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2967 
2968  Value *LHS, *RHS;
2969  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2971  switch (SPR.Flavor) {
2972  case SPF_UMAX: Opc = ISD::UMAX; break;
2973  case SPF_UMIN: Opc = ISD::UMIN; break;
2974  case SPF_SMAX: Opc = ISD::SMAX; break;
2975  case SPF_SMIN: Opc = ISD::SMIN; break;
2976  case SPF_FMINNUM:
2977  switch (SPR.NaNBehavior) {
2978  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2979  case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2980  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2981  case SPNB_RETURNS_ANY: {
2982  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2983  Opc = ISD::FMINNUM;
2984  else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2985  Opc = ISD::FMINNAN;
2986  else if (UseScalarMinMax)
2987  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2989  break;
2990  }
2991  }
2992  break;
2993  case SPF_FMAXNUM:
2994  switch (SPR.NaNBehavior) {
2995  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2996  case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2997  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2998  case SPNB_RETURNS_ANY:
2999 
3000  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3001  Opc = ISD::FMAXNUM;
3002  else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
3003  Opc = ISD::FMAXNAN;
3004  else if (UseScalarMinMax)
3005  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3007  break;
3008  }
3009  break;
3010  default: break;
3011  }
3012 
3013  if (Opc != ISD::DELETED_NODE &&
3014  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3015  (UseScalarMinMax &&
3016  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3017  // If the underlying comparison instruction is used by any other
3018  // instruction, the consumed instructions won't be destroyed, so it is
3019  // not profitable to convert to a min/max.
3020  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3021  OpCode = Opc;
3022  LHSVal = getValue(LHS);
3023  RHSVal = getValue(RHS);
3024  BaseOps = {};
3025  }
3026  }
3027 
3028  for (unsigned i = 0; i != NumValues; ++i) {
3029  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3030  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3031  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3032  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3033  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3034  Ops);
3035  }
3036 
3037  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3038  DAG.getVTList(ValueVTs), Values));
3039 }
3040 
3041 void SelectionDAGBuilder::visitTrunc(const User &I) {
3042  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3043  SDValue N = getValue(I.getOperand(0));
3044  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3045  I.getType());
3046  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3047 }
3048 
3049 void SelectionDAGBuilder::visitZExt(const User &I) {
3050  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3051  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3052  SDValue N = getValue(I.getOperand(0));
3053  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3054  I.getType());
3055  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3056 }
3057 
3058 void SelectionDAGBuilder::visitSExt(const User &I) {
3059  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3060  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3061  SDValue N = getValue(I.getOperand(0));
3062  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3063  I.getType());
3064  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3065 }
3066 
3067 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3068  // FPTrunc is never a no-op cast, no need to check
3069  SDValue N = getValue(I.getOperand(0));
3070  SDLoc dl = getCurSDLoc();
3071  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3072  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3073  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3074  DAG.getTargetConstant(
3075  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3076 }
3077 
3078 void SelectionDAGBuilder::visitFPExt(const User &I) {
3079  // FPExt is never a no-op cast, no need to check
3080  SDValue N = getValue(I.getOperand(0));
3081  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3082  I.getType());
3083  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3084 }
3085 
3086 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3087  // FPToUI is never a no-op cast, no need to check
3088  SDValue N = getValue(I.getOperand(0));
3089  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3090  I.getType());
3091  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3092 }
3093 
3094 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3095  // FPToSI is never a no-op cast, no need to check
3096  SDValue N = getValue(I.getOperand(0));
3097  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3098  I.getType());
3099  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3100 }
3101 
3102 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3103  // UIToFP is never a no-op cast, no need to check
3104  SDValue N = getValue(I.getOperand(0));
3105  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3106  I.getType());
3107  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3108 }
3109 
3110 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3111  // SIToFP is never a no-op cast, no need to check
3112  SDValue N = getValue(I.getOperand(0));
3113  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3114  I.getType());
3115  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3116 }
3117 
3118 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3119  // What to do depends on the size of the integer and the size of the pointer.
3120  // We can either truncate, zero extend, or no-op, accordingly.
3121  SDValue N = getValue(I.getOperand(0));
3122  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3123  I.getType());
3124  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3125 }
3126 
3127 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3128  // What to do depends on the size of the integer and the size of the pointer.
3129  // We can either truncate, zero extend, or no-op, accordingly.
3130  SDValue N = getValue(I.getOperand(0));
3131  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3132  I.getType());
3133  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3134 }
3135 
3136 void SelectionDAGBuilder::visitBitCast(const User &I) {
3137  SDValue N = getValue(I.getOperand(0));
3138  SDLoc dl = getCurSDLoc();
3139  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3140  I.getType());
3141 
3142  // BitCast assures us that source and destination are the same size so this is
3143  // either a BITCAST or a no-op.
3144  if (DestVT != N.getValueType())
3145  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3146  DestVT, N)); // convert types.
3147  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3148  // might fold any kind of constant expression to an integer constant and that
3149  // is not what we are looking for. Only recognize a bitcast of a genuine
3150  // constant integer as an opaque constant.
3151  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3152  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3153  /*isOpaque*/true));
3154  else
3155  setValue(&I, N); // noop cast.
3156 }
3157 
3158 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3159  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3160  const Value *SV = I.getOperand(0);
3161  SDValue N = getValue(SV);
3162  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3163 
3164  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3165  unsigned DestAS = I.getType()->getPointerAddressSpace();
3166 
3167  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3168  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3169 
3170  setValue(&I, N);
3171 }
3172 
3173 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3174  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3175  SDValue InVec = getValue(I.getOperand(0));
3176  SDValue InVal = getValue(I.getOperand(1));
3177  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3178  TLI.getVectorIdxTy(DAG.getDataLayout()));
3179  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3180  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3181  InVec, InVal, InIdx));
3182 }
3183 
3184 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3185  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3186  SDValue InVec = getValue(I.getOperand(0));
3187  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3188  TLI.getVectorIdxTy(DAG.getDataLayout()));
3189  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3190  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3191  InVec, InIdx));
3192 }
3193 
3194 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3195  SDValue Src1 = getValue(I.getOperand(0));
3196  SDValue Src2 = getValue(I.getOperand(1));
3197  SDLoc DL = getCurSDLoc();
3198 
3200  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3201  unsigned MaskNumElts = Mask.size();
3202 
3203  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3204  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3205  EVT SrcVT = Src1.getValueType();
3206  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3207 
3208  if (SrcNumElts == MaskNumElts) {
3209  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3210  return;
3211  }
3212 
3213  // Normalize the shuffle vector since mask and vector length don't match.
3214  if (SrcNumElts < MaskNumElts) {
3215  // Mask is longer than the source vectors. We can use concatenate vector to
3216  // make the mask and vectors lengths match.
3217 
3218  if (MaskNumElts % SrcNumElts == 0) {
3219  // Mask length is a multiple of the source vector length.
3220  // Check if the shuffle is some kind of concatenation of the input
3221  // vectors.
3222  unsigned NumConcat = MaskNumElts / SrcNumElts;
3223  bool IsConcat = true;
3224  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3225  for (unsigned i = 0; i != MaskNumElts; ++i) {
3226  int Idx = Mask[i];
3227  if (Idx < 0)
3228  continue;
3229  // Ensure the indices in each SrcVT sized piece are sequential and that
3230  // the same source is used for the whole piece.
3231  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3232  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3233  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3234  IsConcat = false;
3235  break;
3236  }
3237  // Remember which source this index came from.
3238  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3239  }
3240 
3241  // The shuffle is concatenating multiple vectors together. Just emit
3242  // a CONCAT_VECTORS operation.
3243  if (IsConcat) {
3244  SmallVector<SDValue, 8> ConcatOps;
3245  for (auto Src : ConcatSrcs) {
3246  if (Src < 0)
3247  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3248  else if (Src == 0)
3249  ConcatOps.push_back(Src1);
3250  else
3251  ConcatOps.push_back(Src2);
3252  }
3253  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3254  return;
3255  }
3256  }
3257 
3258  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3259  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3260  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3261  PaddedMaskNumElts);
3262 
3263  // Pad both vectors with undefs to make them the same length as the mask.
3264  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3265 
3266  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3267  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3268  MOps1[0] = Src1;
3269  MOps2[0] = Src2;
3270 
3271  Src1 = Src1.isUndef()
3272  ? DAG.getUNDEF(PaddedVT)
3273  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3274  Src2 = Src2.isUndef()
3275  ? DAG.getUNDEF(PaddedVT)
3276  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3277 
3278  // Readjust mask for new input vector length.
3279  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3280  for (unsigned i = 0; i != MaskNumElts; ++i) {
3281  int Idx = Mask[i];
3282  if (Idx >= (int)SrcNumElts)
3283  Idx -= SrcNumElts - PaddedMaskNumElts;
3284  MappedOps[i] = Idx;
3285  }
3286 
3287  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3288 
3289  // If the concatenated vector was padded, extract a subvector with the
3290  // correct number of elements.
3291  if (MaskNumElts != PaddedMaskNumElts)
3292  Result = DAG.getNode(
3293  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3294  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3295 
3296  setValue(&I, Result);
3297  return;
3298  }
3299 
3300  if (SrcNumElts > MaskNumElts) {
3301  // Analyze the access pattern of the vector to see if we can extract
3302  // two subvectors and do the shuffle.
3303  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3304  bool CanExtract = true;
3305  for (int Idx : Mask) {
3306  unsigned Input = 0;
3307  if (Idx < 0)
3308  continue;
3309 
3310  if (Idx >= (int)SrcNumElts) {
3311  Input = 1;
3312  Idx -= SrcNumElts;
3313  }
3314 
3315  // If all the indices come from the same MaskNumElts sized portion of
3316  // the sources we can use extract. Also make sure the extract wouldn't
3317  // extract past the end of the source.
3318  int NewStartIdx = alignDown(Idx, MaskNumElts);
3319  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3320  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3321  CanExtract = false;
3322  // Make sure we always update StartIdx as we use it to track if all
3323  // elements are undef.
3324  StartIdx[Input] = NewStartIdx;
3325  }
3326 
3327  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3328  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3329  return;
3330  }
3331  if (CanExtract) {
3332  // Extract appropriate subvector and generate a vector shuffle
3333  for (unsigned Input = 0; Input < 2; ++Input) {
3334  SDValue &Src = Input == 0 ? Src1 : Src2;
3335  if (StartIdx[Input] < 0)
3336  Src = DAG.getUNDEF(VT);
3337  else {
3338  Src = DAG.getNode(
3339  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3340  DAG.getConstant(StartIdx[Input], DL,
3341  TLI.getVectorIdxTy(DAG.getDataLayout())));
3342  }
3343  }
3344 
3345  // Calculate new mask.
3346  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3347  for (int &Idx : MappedOps) {
3348  if (Idx >= (int)SrcNumElts)
3349  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3350  else if (Idx >= 0)
3351  Idx -= StartIdx[0];
3352  }
3353 
3354  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3355  return;
3356  }
3357  }
3358 
3359  // We can't use either concat vectors or extract subvectors so fall back to
3360  // replacing the shuffle with extract and build vector.
3361  // to insert and build vector.
3362  EVT EltVT = VT.getVectorElementType();
3363  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3365  for (int Idx : Mask) {
3366  SDValue Res;
3367 
3368  if (Idx < 0) {
3369  Res = DAG.getUNDEF(EltVT);
3370  } else {
3371  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3372  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3373 
3374  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3375  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3376  }
3377 
3378  Ops.push_back(Res);
3379  }
3380 
3381  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3382 }
3383 
3384 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3385  ArrayRef<unsigned> Indices;
3386  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3387  Indices = IV->getIndices();
3388  else
3389  Indices = cast<ConstantExpr>(&I)->getIndices();
3390 
3391  const Value *Op0 = I.getOperand(0);
3392  const Value *Op1 = I.getOperand(1);
3393  Type *AggTy = I.getType();
3394  Type *ValTy = Op1->getType();
3395  bool IntoUndef = isa<UndefValue>(Op0);
3396  bool FromUndef = isa<UndefValue>(Op1);
3397 
3398  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3399 
3400  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3401  SmallVector<EVT, 4> AggValueVTs;
3402  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3403  SmallVector<EVT, 4> ValValueVTs;
3404  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3405 
3406  unsigned NumAggValues = AggValueVTs.size();
3407  unsigned NumValValues = ValValueVTs.size();
3408  SmallVector<SDValue, 4> Values(NumAggValues);
3409 
3410  // Ignore an insertvalue that produces an empty object
3411  if (!NumAggValues) {
3412  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3413  return;
3414  }
3415 
3416  SDValue Agg = getValue(Op0);
3417  unsigned i = 0;
3418  // Copy the beginning value(s) from the original aggregate.
3419  for (; i != LinearIndex; ++i)
3420  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3421  SDValue(Agg.getNode(), Agg.getResNo() + i);
3422  // Copy values from the inserted value(s).
3423  if (NumValValues) {
3424  SDValue Val = getValue(Op1);
3425  for (; i != LinearIndex + NumValValues; ++i)
3426  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3427  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3428  }
3429  // Copy remaining value(s) from the original aggregate.
3430  for (; i != NumAggValues; ++i)
3431  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3432  SDValue(Agg.getNode(), Agg.getResNo() + i);
3433 
3434  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3435  DAG.getVTList(AggValueVTs), Values));
3436 }
3437 
3438 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3439  ArrayRef<unsigned> Indices;
3440  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3441  Indices = EV->getIndices();
3442  else
3443  Indices = cast<ConstantExpr>(&I)->getIndices();
3444 
3445  const Value *Op0 = I.getOperand(0);
3446  Type *AggTy = Op0->getType();
3447  Type *ValTy = I.getType();
3448  bool OutOfUndef = isa<UndefValue>(Op0);
3449 
3450  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3451 
3452  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3453  SmallVector<EVT, 4> ValValueVTs;
3454  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3455 
3456  unsigned NumValValues = ValValueVTs.size();
3457 
3458  // Ignore a extractvalue that produces an empty object
3459  if (!NumValValues) {
3460  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3461  return;
3462  }
3463 
3464  SmallVector<SDValue, 4> Values(NumValValues);
3465 
3466  SDValue Agg = getValue(Op0);
3467  // Copy out the selected value(s).
3468  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3469  Values[i - LinearIndex] =
3470  OutOfUndef ?
3471  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3472  SDValue(Agg.getNode(), Agg.getResNo() + i);
3473 
3474  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3475  DAG.getVTList(ValValueVTs), Values));
3476 }
3477 
3478 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3479  Value *Op0 = I.getOperand(0);
3480  // Note that the pointer operand may be a vector of pointers. Take the scalar
3481  // element which holds a pointer.
3482  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3483  SDValue N = getValue(Op0);
3484  SDLoc dl = getCurSDLoc();
3485 
3486  // Normalize Vector GEP - all scalar operands should be converted to the
3487  // splat vector.
3488  unsigned VectorWidth = I.getType()->isVectorTy() ?
3489  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3490 
3491  if (VectorWidth && !N.getValueType().isVector()) {
3492  LLVMContext &Context = *DAG.getContext();
3493  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3494  N = DAG.getSplatBuildVector(VT, dl, N);
3495  }
3496 
3497  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3498  GTI != E; ++GTI) {
3499  const Value *Idx = GTI.getOperand();
3500  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3501  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3502  if (Field) {
3503  // N = N + Offset
3504  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3505 
3506  // In an inbounds GEP with an offset that is nonnegative even when
3507  // interpreted as signed, assume there is no unsigned overflow.
3508  SDNodeFlags Flags;
3509  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3510  Flags.setNoUnsignedWrap(true);
3511 
3512  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3513  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3514  }
3515  } else {
3516  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3517  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3518  APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3519 
3520  // If this is a scalar constant or a splat vector of constants,
3521  // handle it quickly.
3522  const auto *CI = dyn_cast<ConstantInt>(Idx);
3523  if (!CI && isa<ConstantDataVector>(Idx) &&
3524  cast<ConstantDataVector>(Idx)->getSplatValue())
3525  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3526 
3527  if (CI) {
3528  if (CI->isZero())
3529  continue;
3530  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3531  LLVMContext &Context = *DAG.getContext();
3532  SDValue OffsVal = VectorWidth ?
3533  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3534  DAG.getConstant(Offs, dl, IdxTy);
3535 
3536  // In an inbouds GEP with an offset that is nonnegative even when
3537  // interpreted as signed, assume there is no unsigned overflow.
3538  SDNodeFlags Flags;
3539  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3540  Flags.setNoUnsignedWrap(true);
3541 
3542  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3543  continue;
3544  }
3545 
3546  // N = N + Idx * ElementSize;
3547  SDValue IdxN = getValue(Idx);
3548 
3549  if (!IdxN.getValueType().isVector() && VectorWidth) {
3550  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3551  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3552  }
3553 
3554  // If the index is smaller or larger than intptr_t, truncate or extend
3555  // it.
3556  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3557 
3558  // If this is a multiply by a power of two, turn it into a shl
3559  // immediately. This is a very common case.
3560  if (ElementSize != 1) {
3561  if (ElementSize.isPowerOf2()) {
3562  unsigned Amt = ElementSize.logBase2();
3563  IdxN = DAG.getNode(ISD::SHL, dl,
3564  N.getValueType(), IdxN,
3565  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3566  } else {
3567  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3568  IdxN = DAG.getNode(ISD::MUL, dl,
3569  N.getValueType(), IdxN, Scale);
3570  }
3571  }
3572 
3573  N = DAG.getNode(ISD::ADD, dl,
3574  N.getValueType(), N, IdxN);
3575  }
3576  }
3577 
3578  setValue(&I, N);
3579 }
3580 
3581 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3582  // If this is a fixed sized alloca in the entry block of the function,
3583  // allocate it statically on the stack.
3584  if (FuncInfo.StaticAllocaMap.count(&I))
3585  return; // getValue will auto-populate this.
3586 
3587  SDLoc dl = getCurSDLoc();
3588  Type *Ty = I.getAllocatedType();
3589  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3590  auto &DL = DAG.getDataLayout();
3591  uint64_t TySize = DL.getTypeAllocSize(Ty);
3592  unsigned Align =
3593  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3594 
3595  SDValue AllocSize = getValue(I.getArraySize());
3596 
3597  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3598  if (AllocSize.getValueType() != IntPtr)
3599  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3600 
3601  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3602  AllocSize,
3603  DAG.getConstant(TySize, dl, IntPtr));
3604 
3605  // Handle alignment. If the requested alignment is less than or equal to
3606  // the stack alignment, ignore it. If the size is greater than or equal to
3607  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3608  unsigned StackAlign =
3610  if (Align <= StackAlign)
3611  Align = 0;
3612 
3613  // Round the size of the allocation up to the stack alignment size
3614  // by add SA-1 to the size. This doesn't overflow because we're computing
3615  // an address inside an alloca.
3616  SDNodeFlags Flags;
3617  Flags.setNoUnsignedWrap(true);
3618  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3619  DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3620 
3621  // Mask out the low bits for alignment purposes.
3622  AllocSize =
3623  DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3624  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3625 
3626  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3627  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3628  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3629  setValue(&I, DSA);
3630  DAG.setRoot(DSA.getValue(1));
3631 
3632  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3633 }
3634 
3635 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3636  if (I.isAtomic())
3637  return visitAtomicLoad(I);
3638 
3639  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3640  const Value *SV = I.getOperand(0);
3641  if (TLI.supportSwiftError()) {
3642  // Swifterror values can come from either a function parameter with
3643  // swifterror attribute or an alloca with swifterror attribute.
3644  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3645  if (Arg->hasSwiftErrorAttr())
3646  return visitLoadFromSwiftError(I);
3647  }
3648 
3649  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3650  if (Alloca->isSwiftError())
3651  return visitLoadFromSwiftError(I);
3652  }
3653  }
3654 
3655  SDValue Ptr = getValue(SV);
3656 
3657  Type *Ty = I.getType();
3658 
3659  bool isVolatile = I.isVolatile();
3660  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3661  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3662  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3663  unsigned Alignment = I.getAlignment();
3664 
3665  AAMDNodes AAInfo;
3666  I.getAAMetadata(AAInfo);
3667  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3668 
3671  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3672  unsigned NumValues = ValueVTs.size();
3673  if (NumValues == 0)
3674  return;
3675 
3676  SDValue Root;
3677  bool ConstantMemory = false;
3678  if (isVolatile || NumValues > MaxParallelChains)
3679  // Serialize volatile loads with other side effects.
3680  Root = getRoot();
3681  else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3682  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3683  // Do not serialize (non-volatile) loads of constant memory with anything.
3684  Root = DAG.getEntryNode();
3685  ConstantMemory = true;
3686  } else {
3687  // Do not serialize non-volatile loads against each other.
3688  Root = DAG.getRoot();
3689  }
3690 
3691  SDLoc dl = getCurSDLoc();
3692 
3693  if (isVolatile)
3694  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3695 
3696  // An aggregate load cannot wrap around the address space, so offsets to its
3697  // parts don't wrap either.
3698  SDNodeFlags Flags;
3699  Flags.setNoUnsignedWrap(true);
3700 
3701  SmallVector<SDValue, 4> Values(NumValues);
3702  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3703  EVT PtrVT = Ptr.getValueType();
3704  unsigned ChainI = 0;
3705  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3706  // Serializing loads here may result in excessive register pressure, and
3707  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3708  // could recover a bit by hoisting nodes upward in the chain by recognizing
3709  // they are side-effect free or do not alias. The optimizer should really
3710  // avoid this case by converting large object/array copies to llvm.memcpy
3711  // (MaxParallelChains should always remain as failsafe).
3712  if (ChainI == MaxParallelChains) {
3713  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3714  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3715  makeArrayRef(Chains.data(), ChainI));
3716  Root = Chain;
3717  ChainI = 0;
3718  }
3719  SDValue A = DAG.getNode(ISD::ADD, dl,
3720  PtrVT, Ptr,
3721  DAG.getConstant(Offsets[i], dl, PtrVT),
3722  Flags);
3723  auto MMOFlags = MachineMemOperand::MONone;
3724  if (isVolatile)
3725  MMOFlags |= MachineMemOperand::MOVolatile;
3726  if (isNonTemporal)
3728  if (isInvariant)
3729  MMOFlags |= MachineMemOperand::MOInvariant;
3730  if (isDereferenceable)
3732  MMOFlags |= TLI.getMMOFlags(I);
3733 
3734  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3735  MachinePointerInfo(SV, Offsets[i]), Alignment,
3736  MMOFlags, AAInfo, Ranges);
3737 
3738  Values[i] = L;
3739  Chains[ChainI] = L.getValue(1);
3740  }
3741 
3742  if (!ConstantMemory) {
3743  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3744  makeArrayRef(Chains.data(), ChainI));
3745  if (isVolatile)
3746  DAG.setRoot(Chain);
3747  else
3748  PendingLoads.push_back(Chain);
3749  }
3750 
3751  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3752  DAG.getVTList(ValueVTs), Values));
3753 }
3754 
3755 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3757  "call visitStoreToSwiftError when backend supports swifterror");
3758 
3761  const Value *SrcV = I.getOperand(0);
3763  SrcV->getType(), ValueVTs, &Offsets);
3764  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3765  "expect a single EVT for swifterror");
3766 
3767  SDValue Src = getValue(SrcV);
3768  // Create a virtual register, then update the virtual register.
3769  unsigned VReg; bool CreatedVReg;
3770  std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3771  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3772  // Chain can be getRoot or getControlRoot.
3773  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3774  SDValue(Src.getNode(), Src.getResNo()));
3775  DAG.setRoot(CopyNode);
3776  if (CreatedVReg)
3777  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3778 }
3779 
3780 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3782  "call visitLoadFromSwiftError when backend supports swifterror");
3783 
3784  assert(!I.isVolatile() &&
3785  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3787  "Support volatile, non temporal, invariant for load_from_swift_error");
3788 
3789  const Value *SV = I.getOperand(0);
3790  Type *Ty = I.getType();
3791  AAMDNodes AAInfo;
3792  I.getAAMetadata(AAInfo);
3793  assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
3794  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
3795  "load_from_swift_error should not be constant memory");
3796 
3800  ValueVTs, &Offsets);
3801  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3802  "expect a single EVT for swifterror");
3803 
3804  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3805  SDValue L = DAG.getCopyFromReg(
3806  getRoot(), getCurSDLoc(),
3807  FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3808  ValueVTs[0]);
3809 
3810  setValue(&I, L);
3811 }
3812 
3813 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3814  if (I.isAtomic())
3815  return visitAtomicStore(I);
3816 
3817  const Value *SrcV = I.getOperand(0);
3818  const Value *PtrV = I.getOperand(1);
3819 
3820  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3821  if (TLI.supportSwiftError()) {
3822  // Swifterror values can come from either a function parameter with
3823  // swifterror attribute or an alloca with swifterror attribute.
3824  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3825  if (Arg->hasSwiftErrorAttr())
3826  return visitStoreToSwiftError(I);
3827  }
3828 
3829  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3830  if (Alloca->isSwiftError())
3831  return visitStoreToSwiftError(I);
3832  }
3833  }
3834 
3838  SrcV->getType(), ValueVTs, &Offsets);
3839  unsigned NumValues = ValueVTs.size();
3840  if (NumValues == 0)
3841  return;
3842 
3843  // Get the lowered operands. Note that we do this after
3844  // checking if NumResults is zero, because with zero results
3845  // the operands won't have values in the map.
3846  SDValue Src = getValue(SrcV);
3847  SDValue Ptr = getValue(PtrV);
3848 
3849  SDValue Root = getRoot();
3850  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3851  SDLoc dl = getCurSDLoc();
3852  EVT PtrVT = Ptr.getValueType();
3853  unsigned Alignment = I.getAlignment();
3854  AAMDNodes AAInfo;
3855  I.getAAMetadata(AAInfo);
3856 
3857  auto MMOFlags = MachineMemOperand::MONone;
3858  if (I.isVolatile())
3859  MMOFlags |= MachineMemOperand::MOVolatile;
3860  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3862  MMOFlags |= TLI.getMMOFlags(I);
3863 
3864  // An aggregate load cannot wrap around the address space, so offsets to its
3865  // parts don't wrap either.
3866  SDNodeFlags Flags;
3867  Flags.setNoUnsignedWrap(true);
3868 
3869  unsigned ChainI = 0;
3870  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3871  // See visitLoad comments.
3872  if (ChainI == MaxParallelChains) {
3873  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3874  makeArrayRef(Chains.data(), ChainI));
3875  Root = Chain;
3876  ChainI = 0;
3877  }
3878  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3879  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3880  SDValue St = DAG.getStore(
3881  Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3882  MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3883  Chains[ChainI] = St;
3884  }
3885 
3886  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3887  makeArrayRef(Chains.data(), ChainI));
3888  DAG.setRoot(StoreNode);
3889 }
3890 
3891 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3892  bool IsCompressing) {
3893  SDLoc sdl = getCurSDLoc();
3894 
3895  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3896  unsigned& Alignment) {
3897  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3898  Src0 = I.getArgOperand(0);
3899  Ptr = I.getArgOperand(1);
3900  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3901  Mask = I.getArgOperand(3);
3902  };
3903  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3904  unsigned& Alignment) {
3905  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3906  Src0 = I.getArgOperand(0);
3907  Ptr = I.getArgOperand(1);
3908  Mask = I.getArgOperand(2);
3909  Alignment = 0;
3910  };
3911 
3912  Value *PtrOperand, *MaskOperand, *Src0Operand;
3913  unsigned Alignment;
3914  if (IsCompressing)
3915  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3916  else
3917  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3918 
3919  SDValue Ptr = getValue(PtrOperand);
3920  SDValue Src0 = getValue(Src0Operand);
3921  SDValue Mask = getValue(MaskOperand);
3922 
3923  EVT VT = Src0.getValueType();
3924  if (!Alignment)
3925  Alignment = DAG.getEVTAlignment(VT);
3926 
3927  AAMDNodes AAInfo;
3928  I.getAAMetadata(AAInfo);
3929 
3930  MachineMemOperand *MMO =
3931  DAG.getMachineFunction().
3932  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3934  Alignment, AAInfo);
3935  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3936  MMO, false /* Truncating */,
3937  IsCompressing);
3938  DAG.setRoot(StoreNode);
3939  setValue(&I, StoreNode);
3940 }
3941 
3942 // Get a uniform base for the Gather/Scatter intrinsic.
3943 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3944 // We try to represent it as a base pointer + vector of indices.
3945 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3946 // The first operand of the GEP may be a single pointer or a vector of pointers
3947 // Example:
3948 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3949 // or
3950 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3951 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3952 //
3953 // When the first GEP operand is a single pointer - it is the uniform base we
3954 // are looking for. If first operand of the GEP is a splat vector - we
3955 // extract the splat value and use it as a uniform base.
3956 // In all other cases the function returns 'false'.
3957 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3958  SDValue &Scale, SelectionDAGBuilder* SDB) {
3959  SelectionDAG& DAG = SDB->DAG;
3960  LLVMContext &Context = *DAG.getContext();
3961 
3962  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3964  if (!GEP)
3965  return false;
3966 
3967  const Value *GEPPtr = GEP->getPointerOperand();
3968  if (!GEPPtr->getType()->isVectorTy())
3969  Ptr = GEPPtr;
3970  else if (!(Ptr = getSplatValue(GEPPtr)))
3971  return false;
3972 
3973  unsigned FinalIndex = GEP->getNumOperands() - 1;
3974  Value *IndexVal = GEP->getOperand(FinalIndex);
3975 
3976  // Ensure all the other indices are 0.
3977  for (unsigned i = 1; i < FinalIndex; ++i) {
3978  auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
3979  if (!C || !C->isZero())
3980  return false;
3981  }
3982 
3983  // The operands of the GEP may be defined in another basic block.
3984  // In this case we'll not find nodes for the operands.
3985  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3986  return false;
3987 
3988  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3989  const DataLayout &DL = DAG.getDataLayout();
3990  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
3991  SDB->getCurSDLoc(), TLI.getPointerTy(DL));
3992  Base = SDB->getValue(Ptr);
3993  Index = SDB->getValue(IndexVal);
3994 
3995  if (!Index.getValueType().isVector()) {
3996  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3997  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3998  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3999  }
4000  return true;
4001 }
4002 
4003 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4004  SDLoc sdl = getCurSDLoc();
4005 
4006  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4007  const Value *Ptr = I.getArgOperand(1);
4008  SDValue Src0 = getValue(I.getArgOperand(0));
4009  SDValue Mask = getValue(I.getArgOperand(3));
4010  EVT VT = Src0.getValueType();
4011  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4012  if (!Alignment)
4013  Alignment = DAG.getEVTAlignment(VT);
4014  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4015 
4016  AAMDNodes AAInfo;
4017  I.getAAMetadata(AAInfo);
4018 
4019  SDValue Base;
4020  SDValue Index;
4021  SDValue Scale;
4022  const Value *BasePtr = Ptr;
4023  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4024 
4025  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4027  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4028  MachineMemOperand::MOStore, VT.getStoreSize(),
4029  Alignment, AAInfo);
4030  if (!UniformBase) {
4031  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4032  Index = getValue(Ptr);
4033  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4034  }
4035  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4036  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4037  Ops, MMO);
4038  DAG.setRoot(Scatter);
4039  setValue(&I, Scatter);
4040 }
4041 
4042 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4043  SDLoc sdl = getCurSDLoc();
4044 
4045  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4046  unsigned& Alignment) {
4047  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4048  Ptr = I.getArgOperand(0);
4049  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4050  Mask = I.getArgOperand(2);
4051  Src0 = I.getArgOperand(3);
4052  };
4053  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4054  unsigned& Alignment) {
4055  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4056  Ptr = I.getArgOperand(0);
4057  Alignment = 0;
4058  Mask = I.getArgOperand(1);
4059  Src0 = I.getArgOperand(2);
4060  };
4061 
4062  Value *PtrOperand, *MaskOperand, *Src0Operand;
4063  unsigned Alignment;
4064  if (IsExpanding)
4065  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4066  else
4067  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4068 
4069  SDValue Ptr = getValue(PtrOperand);
4070  SDValue Src0 = getValue(Src0Operand);
4071  SDValue Mask = getValue(MaskOperand);
4072 
4073  EVT VT = Src0.getValueType();
4074  if (!Alignment)
4075  Alignment = DAG.getEVTAlignment(VT);
4076 
4077  AAMDNodes AAInfo;
4078  I.getAAMetadata(AAInfo);
4079  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4080 
4081  // Do not serialize masked loads of constant memory with anything.
4082  bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
4083  PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
4084  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4085 
4086  MachineMemOperand *MMO =
4087  DAG.getMachineFunction().
4088  getMachineMemOperand(MachinePointerInfo(PtrOperand),
4090  Alignment, AAInfo, Ranges);
4091 
4092  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4093  ISD::NON_EXTLOAD, IsExpanding);
4094  if (AddToChain)
4095  PendingLoads.push_back(Load.getValue(1));
4096  setValue(&I, Load);
4097 }
4098 
4099 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4100  SDLoc sdl = getCurSDLoc();
4101 
4102  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4103  const Value *Ptr = I.getArgOperand(0);
4104  SDValue Src0 = getValue(I.getArgOperand(3));
4105  SDValue Mask = getValue(I.getArgOperand(2));
4106 
4107  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4108  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4109  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4110  if (!Alignment)
4111  Alignment = DAG.getEVTAlignment(VT);
4112 
4113  AAMDNodes AAInfo;
4114  I.getAAMetadata(AAInfo);
4115  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4116 
4117  SDValue Root = DAG.getRoot();
4118  SDValue Base;
4119  SDValue Index;
4120  SDValue Scale;
4121  const Value *BasePtr = Ptr;
4122  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4123  bool ConstantMemory = false;
4124  if (UniformBase &&
4125  AA && AA->pointsToConstantMemory(MemoryLocation(
4126  BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
4127  AAInfo))) {
4128  // Do not serialize (non-volatile) loads of constant memory with anything.
4129  Root = DAG.getEntryNode();
4130  ConstantMemory = true;
4131  }
4132 
4133  MachineMemOperand *MMO =
4134  DAG.getMachineFunction().
4135  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4137  Alignment, AAInfo, Ranges);
4138 
4139  if (!UniformBase) {
4140  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4141  Index = getValue(Ptr);
4142  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4143  }
4144  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4145  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4146  Ops, MMO);
4147 
4148  SDValue OutChain = Gather.getValue(1);
4149  if (!ConstantMemory)
4150  PendingLoads.push_back(OutChain);
4151  setValue(&I, Gather);
4152 }
4153 
4154 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4155  SDLoc dl = getCurSDLoc();
4156  AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4157  AtomicOrdering FailureOrder = I.getFailureOrdering();
4158  SyncScope::ID SSID = I.getSyncScopeID();
4159 
4160  SDValue InChain = getRoot();
4161 
4162  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4163  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4164  SDValue L = DAG.getAtomicCmpSwap(
4165  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4166  getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4168  /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4169 
4170  SDValue OutChain = L.getValue(2);
4171 
4172  setValue(&I, L);
4173  DAG.setRoot(OutChain);
4174 }
4175 
4176 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4177  SDLoc dl = getCurSDLoc();
4178  ISD::NodeType NT;
4179  switch (I.getOperation()) {
4180  default: llvm_unreachable("Unknown atomicrmw operation");
4181  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4182  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4183  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4184  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4185  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4186  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4187  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4188  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4189  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4190  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4191  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4192  }
4193  AtomicOrdering Order = I.getOrdering();
4194  SyncScope::ID SSID = I.getSyncScopeID();
4195 
4196  SDValue InChain = getRoot();
4197 
4198  SDValue L =
4199  DAG.getAtomic(NT, dl,
4200  getValue(I.getValOperand()).getSimpleValueType(),
4201  InChain,
4202  getValue(I.getPointerOperand()),
4203  getValue(I.getValOperand()),
4204  I.getPointerOperand(),
4205  /* Alignment=*/ 0, Order, SSID);
4206 
4207  SDValue OutChain = L.getValue(1);
4208 
4209  setValue(&I, L);
4210  DAG.setRoot(OutChain);
4211 }
4212 
4213 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4214  SDLoc dl = getCurSDLoc();
4215  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4216  SDValue Ops[3];
4217  Ops[0] = getRoot();
4218  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4219  TLI.getFenceOperandTy(DAG.getDataLayout()));
4220  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4221  TLI.getFenceOperandTy(DAG.getDataLayout()));
4222  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4223 }
4224 
4225 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4226  SDLoc dl = getCurSDLoc();
4227  AtomicOrdering Order = I.getOrdering();
4228  SyncScope::ID SSID = I.getSyncScopeID();
4229 
4230  SDValue InChain = getRoot();
4231 
4232  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4233  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4234 
4235  if (!TLI.supportsUnalignedAtomics() &&
4236  I.getAlignment() < VT.getStoreSize())
4237  report_fatal_error("Cannot generate unaligned atomic load");
4238 
4239  MachineMemOperand *MMO =
4240  DAG.getMachineFunction().
4241  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4244  VT.getStoreSize(),
4245  I.getAlignment() ? I.getAlignment() :
4246  DAG.getEVTAlignment(VT),
4247  AAMDNodes(), nullptr, SSID, Order);
4248 
4249  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4250  SDValue L =
4251  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4252  getValue(I.getPointerOperand()), MMO);
4253 
4254  SDValue OutChain = L.getValue(1);
4255 
4256  setValue(&I, L);
4257  DAG.setRoot(OutChain);
4258 }
4259 
4260 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4261  SDLoc dl = getCurSDLoc();
4262 
4263  AtomicOrdering Order = I.getOrdering();
4264  SyncScope::ID SSID = I.getSyncScopeID();
4265 
4266  SDValue InChain = getRoot();
4267 
4268  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4269  EVT VT =
4271 
4272  if (I.getAlignment() < VT.getStoreSize())
4273  report_fatal_error("Cannot generate unaligned atomic store");
4274 
4275  SDValue OutChain =
4276  DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4277  InChain,
4278  getValue(I.getPointerOperand()),
4279  getValue(I.getValueOperand()),
4281  Order, SSID);
4282 
4283  DAG.setRoot(OutChain);
4284 }
4285 
4286 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4287 /// node.
4288 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4289  unsigned Intrinsic) {
4290  // Ignore the callsite's attributes. A specific call site may be marked with
4291  // readnone, but the lowering code will expect the chain based on the
4292  // definition.
4293  const Function *F = I.getCalledFunction();
4294  bool HasChain = !F->doesNotAccessMemory();
4295  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4296 
4297  // Build the operand list.
4299  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4300  if (OnlyLoad) {
4301  // We don't need to serialize loads against other loads.
4302  Ops.push_back(DAG.getRoot());
4303  } else {
4304  Ops.push_back(getRoot());
4305  }
4306  }
4307 
4308  // Info is set by getTgtMemInstrinsic
4309  TargetLowering::IntrinsicInfo Info;
4310  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4311  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4312  DAG.getMachineFunction(),
4313  Intrinsic);
4314 
4315  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4316  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4317  Info.opc == ISD::INTRINSIC_W_CHAIN)
4318  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4319  TLI.getPointerTy(DAG.getDataLayout())));
4320 
4321  // Add all operands of the call to the operand list.
4322  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4323  SDValue Op = getValue(I.getArgOperand(i));
4324  Ops.push_back(Op);
4325  }
4326 
4328  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4329 
4330  if (HasChain)
4331  ValueVTs.push_back(MVT::Other);
4332 
4333  SDVTList VTs = DAG.getVTList(ValueVTs);
4334 
4335  // Create the node.
4336  SDValue Result;
4337  if (IsTgtIntrinsic) {
4338  // This is target intrinsic that touches memory
4339  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4340  Ops, Info.memVT,
4341  MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4342  Info.flags, Info.size);
4343  } else if (!HasChain) {
4344  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4345  } else if (!I.getType()->isVoidTy()) {
4346  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4347  } else {
4348  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4349  }
4350 
4351  if (HasChain) {
4352  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4353  if (OnlyLoad)
4354  PendingLoads.push_back(Chain);
4355  else
4356  DAG.setRoot(Chain);
4357  }
4358 
4359  if (!I.getType()->isVoidTy()) {
4360  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4361  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4362  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4363  } else
4364  Result = lowerRangeToAssertZExt(DAG, I, Result);
4365 
4366  setValue(&I, Result);
4367  }
4368 }
4369 
4370 /// GetSignificand - Get the significand and build it into a floating-point
4371 /// number with exponent of 1:
4372 ///
4373 /// Op = (Op & 0x007fffff) | 0x3f800000;
4374 ///
4375 /// where Op is the hexadecimal representation of floating point value.
4377  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4378  DAG.getConstant(0x007fffff, dl, MVT::i32));
4379  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4380  DAG.getConstant(0x3f800000, dl, MVT::i32));
4381  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4382 }
4383 
4384 /// GetExponent - Get the exponent:
4385 ///
4386 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4387 ///
4388 /// where Op is the hexadecimal representation of floating point value.
4390  const TargetLowering &TLI, const SDLoc &dl) {
4391  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4392  DAG.getConstant(0x7f800000, dl, MVT::i32));
4393  SDValue t1 = DAG.getNode(
4394  ISD::SRL, dl, MVT::i32, t0,
4395  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4396  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4397  DAG.getConstant(127, dl, MVT::i32));
4398  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4399 }
4400 
4401 /// getF32Constant - Get 32-bit floating point constant.
4402 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4403  const SDLoc &dl) {
4404  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4405  MVT::f32);
4406 }
4407 
4409  SelectionDAG &DAG) {
4410  // TODO: What fast-math-flags should be set on the floating-point nodes?
4411 
4412  // IntegerPartOfX = ((int32_t)(t0);
4413  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4414 
4415  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4416  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4417  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4418 
4419  // IntegerPartOfX <<= 23;
4420  IntegerPartOfX = DAG.getNode(
4421  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4423  DAG.getDataLayout())));
4424 
4425  SDValue TwoToFractionalPartOfX;
4426  if (LimitFloatPrecision <= 6) {
4427  // For floating-point precision of 6:
4428  //
4429  // TwoToFractionalPartOfX =
4430  // 0.997535578f +
4431  // (0.735607626f + 0.252464424f * x) * x;
4432  //
4433  // error 0.0144103317, which is 6 bits
4434  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4435  getF32Constant(DAG, 0x3e814304, dl));
4436  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4437  getF32Constant(DAG, 0x3f3c50c8, dl));
4438  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4439  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4440  getF32Constant(DAG, 0x3f7f5e7e, dl));
4441  } else if (LimitFloatPrecision <= 12) {
4442  // For floating-point precision of 12:
4443  //
4444  // TwoToFractionalPartOfX =
4445  // 0.999892986f +
4446  // (0.696457318f +
4447  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4448  //
4449  // error 0.000107046256, which is 13 to 14 bits
4450  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4451  getF32Constant(DAG, 0x3da235e3, dl));
4452  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4453  getF32Constant(DAG, 0x3e65b8f3, dl));
4454  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4455  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4456  getF32Constant(DAG, 0x3f324b07, dl));
4457  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4458  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4459  getF32Constant(DAG, 0x3f7ff8fd, dl));
4460  } else { // LimitFloatPrecision <= 18
4461  // For floating-point precision of 18:
4462  //
4463  // TwoToFractionalPartOfX =
4464  // 0.999999982f +
4465  // (0.693148872f +
4466  // (0.240227044f +
4467  // (0.554906021e-1f +
4468  // (0.961591928e-2f +
4469  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4470  // error 2.47208000*10^(-7), which is better than 18 bits
4471  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4472  getF32Constant(DAG, 0x3924b03e, dl));
4473  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4474  getF32Constant(DAG, 0x3ab24b87, dl));
4475  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4476  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4477  getF32Constant(DAG, 0x3c1d8c17, dl));
4478  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4479  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4480  getF32Constant(DAG, 0x3d634a1d, dl));
4481  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4482  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4483  getF32Constant(DAG, 0x3e75fe14, dl));
4484  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4485  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4486  getF32Constant(DAG, 0x3f317234, dl));
4487  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4488  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4489  getF32Constant(DAG, 0x3f800000, dl));
4490  }
4491 
4492  // Add the exponent into the result in integer domain.
4493  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4494  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4495  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4496 }
4497 
4498 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4499 /// limited-precision mode.
4500 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4501  const TargetLowering &TLI) {
4502  if (Op.getValueType() == MVT::f32 &&
4503  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4504 
4505  // Put the exponent in the right bit position for later addition to the
4506  // final result:
4507  //
4508  // #define LOG2OFe 1.4426950f
4509  // t0 = Op * LOG2OFe
4510 
4511  // TODO: What fast-math-flags should be set here?
4512  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4513  getF32Constant(DAG, 0x3fb8aa3b, dl));
4514  return getLimitedPrecisionExp2(t0, dl, DAG);
4515  }
4516 
4517  // No special expansion.
4518  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4519 }
4520 
4521 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4522 /// limited-precision mode.
4523 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4524  const TargetLowering &TLI) {
4525  // TODO: What fast-math-flags should be set on the floating-point nodes?
4526 
4527  if (Op.getValueType() == MVT::f32 &&
4528  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4529  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4530 
4531  // Scale the exponent by log(2) [0.69314718f].
4532  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4533  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4534  getF32Constant(DAG, 0x3f317218, dl));
4535 
4536  // Get the significand and build it into a floating-point number with
4537  // exponent of 1.
4538  SDValue X = GetSignificand(DAG, Op1, dl);
4539 
4540  SDValue LogOfMantissa;
4541  if (LimitFloatPrecision <= 6) {
4542  // For floating-point precision of 6:
4543  //
4544  // LogofMantissa =
4545  // -1.1609546f +
4546  // (1.4034025f - 0.23903021f * x) * x;
4547  //
4548  // error 0.0034276066, which is better than 8 bits
4549  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4550  getF32Constant(DAG, 0xbe74c456, dl));
4551  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4552  getF32Constant(DAG, 0x3fb3a2b1, dl));
4553  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4554  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4555  getF32Constant(DAG, 0x3f949a29, dl));
4556  } else if (LimitFloatPrecision <= 12) {
4557  // For floating-point precision of 12:
4558  //
4559  // LogOfMantissa =
4560  // -1.7417939f +
4561  // (2.8212026f +
4562  // (-1.4699568f +
4563  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4564  //
4565  // error 0.000061011436, which is 14 bits
4566  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4567  getF32Constant(DAG, 0xbd67b6d6, dl));
4568  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4569  getF32Constant(DAG, 0x3ee4f4b8, dl));
4570  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4571  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4572  getF32Constant(DAG, 0x3fbc278b, dl));
4573  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4574  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4575  getF32Constant(DAG, 0x40348e95, dl));
4576  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4577  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4578  getF32Constant(DAG, 0x3fdef31a, dl));
4579  } else { // LimitFloatPrecision <= 18
4580  // For floating-point precision of 18:
4581  //
4582  // LogOfMantissa =
4583  // -2.1072184f +
4584  // (4.2372794f +
4585  // (-3.7029485f +
4586  // (2.2781945f +
4587  // (-0.87823314f +
4588  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4589  //
4590  // error 0.0000023660568, which is better than 18 bits
4591  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4592  getF32Constant(DAG, 0xbc91e5ac, dl));
4593  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4594  getF32Constant(DAG, 0x3e4350aa, dl));
4595  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4596  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4597  getF32Constant(DAG, 0x3f60d3e3, dl));
4598  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4599  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4600  getF32Constant(DAG, 0x4011cdf0, dl));
4601  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4602  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4603  getF32Constant(DAG, 0x406cfd1c, dl));
4604  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4605  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4606  getF32Constant(DAG, 0x408797cb, dl));
4607  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4608  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4609  getF32Constant(DAG, 0x4006dcab, dl));
4610  }
4611 
4612  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4613  }
4614 
4615  // No special expansion.
4616  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4617 }
4618 
4619 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4620 /// limited-precision mode.
4621 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4622  const TargetLowering &TLI) {
4623  // TODO: What fast-math-flags should be set on the floating-point nodes?
4624 
4625  if (Op.getValueType() == MVT::f32 &&
4626  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4627  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4628 
4629  // Get the exponent.
4630  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4631 
4632  // Get the significand and build it into a floating-point number with
4633  // exponent of 1.
4634  SDValue X = GetSignificand(DAG, Op1, dl);
4635 
4636  // Different possible minimax approximations of significand in
4637  // floating-point for various degrees of accuracy over [1,2].
4638  SDValue Log2ofMantissa;
4639  if (LimitFloatPrecision <= 6) {
4640  // For floating-point precision of 6:
4641  //
4642  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4643  //
4644  // error 0.0049451742, which is more than 7 bits
4645  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4646  getF32Constant(DAG, 0xbeb08fe0, dl));
4647  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4648  getF32Constant(DAG, 0x40019463, dl));
4649  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4650  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4651  getF32Constant(DAG, 0x3fd6633d, dl));
4652  } else if (LimitFloatPrecision <= 12) {
4653  // For floating-point precision of 12:
4654  //
4655  // Log2ofMantissa =
4656  // -2.51285454f +
4657  // (4.07009056f +
4658  // (-2.12067489f +
4659  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4660  //
4661  // error 0.0000876136000, which is better than 13 bits
4662  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4663  getF32Constant(DAG, 0xbda7262e, dl));
4664  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4665  getF32Constant(DAG, 0x3f25280b, dl));
4666  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4667  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4668  getF32Constant(DAG, 0x4007b923, dl));
4669  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4670  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4671  getF32Constant(DAG, 0x40823e2f, dl));
4672  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4673  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4674  getF32Constant(DAG, 0x4020d29c, dl));
4675  } else { // LimitFloatPrecision <= 18
4676  // For floating-point precision of 18:
4677  //
4678  // Log2ofMantissa =
4679  // -3.0400495f +
4680  // (6.1129976f +
4681  // (-5.3420409f +
4682  // (3.2865683f +
4683  // (-1.2669343f +
4684  // (0.27515199f -
4685  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4686  //
4687  // error 0.0000018516, which is better than 18 bits
4688  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4689  getF32Constant(DAG, 0xbcd2769e, dl));
4690  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4691  getF32Constant(DAG, 0x3e8ce0b9, dl));
4692  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4693  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4694  getF32Constant(DAG, 0x3fa22ae7, dl));
4695  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4696  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4697  getF32Constant(DAG, 0x40525723, dl));
4698  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4699  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4700  getF32Constant(DAG, 0x40aaf200, dl));
4701  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4702  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4703  getF32Constant(DAG, 0x40c39dad, dl));
4704  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4705  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4706  getF32Constant(DAG, 0x4042902c, dl));
4707  }
4708 
4709  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4710  }
4711 
4712  // No special expansion.
4713  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4714 }
4715 
4716 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4717 /// limited-precision mode.
4719  const TargetLowering &TLI) {
4720  // TODO: What fast-math-flags should be set on the floating-point nodes?
4721 
4722  if (Op.getValueType() == MVT::f32 &&
4723  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4724  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4725 
4726  // Scale the exponent by log10(2) [0.30102999f].
4727  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4728  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4729  getF32Constant(DAG, 0x3e9a209a, dl));
4730 
4731  // Get the significand and build it into a floating-point number with
4732  // exponent of 1.
4733  SDValue X = GetSignificand(DAG, Op1, dl);
4734 
4735  SDValue Log10ofMantissa;
4736  if (LimitFloatPrecision <= 6) {
4737  // For floating-point precision of 6:
4738  //
4739  // Log10ofMantissa =
4740  // -0.50419619f +
4741  // (0.60948995f - 0.10380950f * x) * x;
4742  //
4743  // error 0.0014886165, which is 6 bits
4744  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4745  getF32Constant(DAG, 0xbdd49a13, dl));
4746  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4747  getF32Constant(DAG, 0x3f1c0789, dl));
4748  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4749  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4750  getF32Constant(DAG, 0x3f011300, dl));
4751  } else if (LimitFloatPrecision <= 12) {
4752  // For floating-point precision of 12:
4753  //
4754  // Log10ofMantissa =
4755  // -0.64831180f +
4756  // (0.91751397f +
4757  // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4758  //
4759  // error 0.00019228036, which is better than 12 bits
4760  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4761  getF32Constant(DAG, 0x3d431f31, dl));
4762  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4763  getF32Constant(DAG, 0x3ea21fb2, dl));
4764  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4765  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4766  getF32Constant(DAG, 0x3f6ae232, dl));
4767  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4768  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4769  getF32Constant(DAG, 0x3f25f7c3, dl));
4770  } else { // LimitFloatPrecision <= 18
4771  // For floating-point precision of 18:
4772  //
4773  // Log10ofMantissa =
4774  // -0.84299375f +
4775  // (1.5327582f +
4776  // (-1.0688956f +
4777  // (0.49102474f +
4778  // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4779  //
4780  // error 0.0000037995730, which is better than 18 bits
4781  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4782  getF32Constant(DAG, 0x3c5d51ce, dl));
4783  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4784  getF32Constant(DAG, 0x3e00685a, dl));
4785  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4786  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4787  getF32Constant(DAG, 0x3efb6798, dl));
4788  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4789  SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4790  getF32Constant(DAG, 0x3f88d192, dl));
4791  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4792  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4793  getF32Constant(DAG, 0x3fc4316c, dl));
4794  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4795  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4796  getF32Constant(DAG, 0x3f57ce70, dl));
4797  }
4798 
4799  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4800  }
4801 
4802  // No special expansion.
4803  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4804 }
4805 
4806 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4807 /// limited-precision mode.
4808 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4809  const TargetLowering &TLI) {
4810  if (Op.getValueType() == MVT::f32 &&
4811  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4812  return getLimitedPrecisionExp2(Op, dl, DAG);
4813 
4814  // No special expansion.
4815  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4816 }
4817 
4818 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4819 /// limited-precision mode with x == 10.0f.
4820 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4821  SelectionDAG &DAG, const TargetLowering &TLI) {
4822  bool IsExp10 = false;
4823  if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4824  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4825  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4826  APFloat Ten(10.0f);
4827  IsExp10 = LHSC->isExactlyValue(Ten);
4828  }
4829  }
4830 
4831  // TODO: What fast-math-flags should be set on the FMUL node?
4832  if (IsExp10) {
4833  // Put the exponent in the right bit position for later addition to the
4834  // final result:
4835  //
4836  // #define LOG2OF10 3.3219281f
4837  // t0 = Op * LOG2OF10;
4838  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4839  getF32Constant(DAG, 0x40549a78, dl));
4840  return getLimitedPrecisionExp2(t0, dl, DAG);
4841  }
4842 
4843  // No special expansion.
4844  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4845 }
4846 
4847 /// ExpandPowI - Expand a llvm.powi intrinsic.
4848 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4849  SelectionDAG &DAG) {
4850  // If RHS is a constant, we can expand this out to a multiplication tree,
4851  // otherwise we end up lowering to a call to __powidf2 (for example). When
4852  // optimizing for size, we only want to do this if the expansion would produce
4853  // a small number of multiplies, otherwise we do the full expansion.
4854  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4855  // Get the exponent as a positive value.
4856  unsigned Val = RHSC->getSExtValue();
4857  if ((int)Val < 0) Val = -Val;
4858 
4859  // powi(x, 0) -> 1.0
4860  if (Val == 0)
4861  return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4862 
4863  const Function &F = DAG.getMachineFunction().getFunction();
4864  if (!F.optForSize() ||
4865  // If optimizing for size, don't insert too many multiplies.
4866  // This inserts up to 5 multiplies.
4867  countPopulation(Val) + Log2_32(Val) < 7) {
4868  // We use the simple binary decomposition method to generate the multiply
4869  // sequence. There are more optimal ways to do this (for example,
4870  // powi(x,15) generates one more multiply than it should), but this has
4871  // the benefit of being both really simple and much better than a libcall.
4872  SDValue Res; // Logically starts equal to 1.0
4873  SDValue CurSquare = LHS;
4874  // TODO: Intrinsics should have fast-math-flags that propagate to these
4875  // nodes.
4876  while (Val) {
4877  if (Val & 1) {
4878  if (Res.getNode())
4879  Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4880  else
4881  Res = CurSquare; // 1.0*CurSquare.
4882  }
4883 
4884  CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4885  CurSquare, CurSquare);
4886  Val >>= 1;
4887  }
4888 
4889  // If the original was negative, invert the result, producing 1/(x*x*x).
4890  if (RHSC->getSExtValue() < 0)
4891  Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4892  DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4893  return Res;
4894  }
4895  }
4896 
4897  // Otherwise, expand to a libcall.
4898  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4899 }
4900 
4901 // getUnderlyingArgReg - Find underlying register used for a truncated or
4902 // bitcasted argument.
4903 static unsigned getUnderlyingArgReg(const SDValue &N) {
4904  switch (N.getOpcode()) {
4905  case ISD::CopyFromReg:
4906  return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4907  case ISD::BITCAST:
4908  case ISD::AssertZext:
4909  case ISD::AssertSext:
4910  case ISD::TRUNCATE:
4911  return getUnderlyingArgReg(N.getOperand(0));
4912  default:
4913  return 0;
4914  }
4915 }
4916 
4917 /// If the DbgValueInst is a dbg_value of a function argument, create the
4918 /// corresponding DBG_VALUE machine instruction for it now. At the end of
4919 /// instruction selection, they will be inserted to the entry BB.
4920 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4921  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4922  DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4923  const Argument *Arg = dyn_cast<Argument>(V);
4924  if (!Arg)
4925  return false;
4926 
4927  MachineFunction &MF = DAG.getMachineFunction();
4928  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4929 
4930  bool IsIndirect = false;
4932  // Some arguments' frame index is recorded during argument lowering.
4933  int FI = FuncInfo.getArgumentFrameIndex(Arg);
4934  if (FI != std::numeric_limits<int>::max())
4935  Op = MachineOperand::CreateFI(FI);
4936 
4937  if (!Op && N.getNode()) {
4938  unsigned Reg = getUnderlyingArgReg(N);
4939  if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4940  MachineRegisterInfo &RegInfo = MF.getRegInfo();
4941  unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4942  if (PR)
4943  Reg = PR;
4944  }
4945  if (Reg) {
4946  Op = MachineOperand::CreateReg(Reg, false);
4947  IsIndirect = IsDbgDeclare;
4948  }
4949  }
4950 
4951  if (!Op && N.getNode())
4952  // Check if frame index is available.
4953  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4954  if (FrameIndexSDNode *FINode =
4955  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4956  Op = MachineOperand::CreateFI(FINode->getIndex());
4957 
4958  if (!Op) {
4959  // Check if ValueMap has reg number.
4960  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4961  if (VMI != FuncInfo.ValueMap.end()) {
4962  const auto &TLI = DAG.getTargetLoweringInfo();
4963  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4964  V->getType(), getABIRegCopyCC(V));
4965  if (RFV.occupiesMultipleRegs()) {
4966  unsigned Offset = 0;
4967  for (auto RegAndSize : RFV.getRegsAndSizes()) {
4968  Op = MachineOperand::CreateReg(RegAndSize.first, false);
4969  auto FragmentExpr = DIExpression::createFragmentExpression(
4970  Expr, Offset, RegAndSize.second);
4971  if (!FragmentExpr)
4972  continue;
4973  FuncInfo.ArgDbgValues.push_back(
4974  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
4975  Op->getReg(), Variable, *FragmentExpr));
4976  Offset += RegAndSize.second;
4977  }
4978  return true;
4979  }
4980  Op = MachineOperand::CreateReg(VMI->second, false);
4981  IsIndirect = IsDbgDeclare;
4982  }
4983  }
4984 
4985  if (!Op)
4986  return false;
4987 
4988  assert(Variable->isValidLocationForIntrinsic(DL) &&
4989  "Expected inlined-at fields to agree");