LLVM  9.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/BitVector.h"
19 #include "llvm/ADT/DenseMap.h"
20 #include "llvm/ADT/None.h"
21 #include "llvm/ADT/Optional.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/ADT/Twine.h"
33 #include "llvm/Analysis/Loads.h"
38 #include "llvm/CodeGen/Analysis.h"
56 #include "llvm/CodeGen/StackMaps.h"
65 #include "llvm/IR/Argument.h"
66 #include "llvm/IR/Attributes.h"
67 #include "llvm/IR/BasicBlock.h"
68 #include "llvm/IR/CFG.h"
69 #include "llvm/IR/CallSite.h"
70 #include "llvm/IR/CallingConv.h"
71 #include "llvm/IR/Constant.h"
72 #include "llvm/IR/ConstantRange.h"
73 #include "llvm/IR/Constants.h"
74 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugLoc.h"
77 #include "llvm/IR/DerivedTypes.h"
78 #include "llvm/IR/Function.h"
80 #include "llvm/IR/InlineAsm.h"
81 #include "llvm/IR/InstrTypes.h"
82 #include "llvm/IR/Instruction.h"
83 #include "llvm/IR/Instructions.h"
84 #include "llvm/IR/IntrinsicInst.h"
85 #include "llvm/IR/Intrinsics.h"
86 #include "llvm/IR/LLVMContext.h"
87 #include "llvm/IR/Metadata.h"
88 #include "llvm/IR/Module.h"
89 #include "llvm/IR/Operator.h"
90 #include "llvm/IR/PatternMatch.h"
91 #include "llvm/IR/Statepoint.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCSymbol.h"
99 #include "llvm/Support/Casting.h"
100 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/Compiler.h"
103 #include "llvm/Support/Debug.h"
106 #include "llvm/Support/MathExtras.h"
112 #include <algorithm>
113 #include <cassert>
114 #include <cstddef>
115 #include <cstdint>
116 #include <cstring>
117 #include <iterator>
118 #include <limits>
119 #include <numeric>
120 #include <tuple>
121 #include <utility>
122 #include <vector>
123 
124 using namespace llvm;
125 using namespace PatternMatch;
126 
127 #define DEBUG_TYPE "isel"
128 
129 /// LimitFloatPrecision - Generate low-precision inline sequences for
130 /// some float libcalls (6, 8 or 12 bits).
131 static unsigned LimitFloatPrecision;
132 
134  LimitFPPrecision("limit-float-precision",
135  cl::desc("Generate low-precision inline sequences "
136  "for some float libcalls"),
137  cl::location(LimitFloatPrecision), cl::Hidden,
138  cl::init(0));
139 
141  "switch-peel-threshold", cl::Hidden, cl::init(66),
142  cl::desc("Set the case probability threshold for peeling the case from a "
143  "switch statement. A value greater than 100 will void this "
144  "optimization"));
145 
146 // Limit the width of DAG chains. This is important in general to prevent
147 // DAG-based analysis from blowing up. For example, alias analysis and
148 // load clustering may not complete in reasonable time. It is difficult to
149 // recognize and avoid this situation within each individual analysis, and
150 // future analyses are likely to have the same behavior. Limiting DAG width is
151 // the safe approach and will be especially important with global DAGs.
152 //
153 // MaxParallelChains default is arbitrarily high to avoid affecting
154 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
155 // sequence over this should have been converted to llvm.memcpy by the
156 // frontend. It is easy to induce this behavior with .ll code such as:
157 // %buffer = alloca [4096 x i8]
158 // %data = load [4096 x i8]* %argPtr
159 // store [4096 x i8] %data, [4096 x i8]* %buffer
160 static const unsigned MaxParallelChains = 64;
161 
162 // Return the calling convention if the Value passed requires ABI mangling as it
163 // is a parameter to a function or a return value from a function which is not
164 // an intrinsic.
166  if (auto *R = dyn_cast<ReturnInst>(V))
167  return R->getParent()->getParent()->getCallingConv();
168 
169  if (auto *CI = dyn_cast<CallInst>(V)) {
170  const bool IsInlineAsm = CI->isInlineAsm();
171  const bool IsIndirectFunctionCall =
172  !IsInlineAsm && !CI->getCalledFunction();
173 
174  // It is possible that the call instruction is an inline asm statement or an
175  // indirect function call in which case the return value of
176  // getCalledFunction() would be nullptr.
177  const bool IsInstrinsicCall =
178  !IsInlineAsm && !IsIndirectFunctionCall &&
179  CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
180 
181  if (!IsInlineAsm && !IsInstrinsicCall)
182  return CI->getCallingConv();
183  }
184 
185  return None;
186 }
187 
188 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
189  const SDValue *Parts, unsigned NumParts,
190  MVT PartVT, EVT ValueVT, const Value *V,
192 
193 /// getCopyFromParts - Create a value that contains the specified legal parts
194 /// combined into the value they represent. If the parts combine to a type
195 /// larger than ValueVT then AssertOp can be used to specify whether the extra
196 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
197 /// (ISD::AssertSext).
199  const SDValue *Parts, unsigned NumParts,
200  MVT PartVT, EVT ValueVT, const Value *V,
201  Optional<CallingConv::ID> CC = None,
202  Optional<ISD::NodeType> AssertOp = None) {
203  if (ValueVT.isVector())
204  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
205  CC);
206 
207  assert(NumParts > 0 && "No parts to assemble!");
208  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
209  SDValue Val = Parts[0];
210 
211  if (NumParts > 1) {
212  // Assemble the value from multiple parts.
213  if (ValueVT.isInteger()) {
214  unsigned PartBits = PartVT.getSizeInBits();
215  unsigned ValueBits = ValueVT.getSizeInBits();
216 
217  // Assemble the power of 2 part.
218  unsigned RoundParts = NumParts & (NumParts - 1) ?
219  1 << Log2_32(NumParts) : NumParts;
220  unsigned RoundBits = PartBits * RoundParts;
221  EVT RoundVT = RoundBits == ValueBits ?
222  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
223  SDValue Lo, Hi;
224 
225  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
226 
227  if (RoundParts > 2) {
228  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
229  PartVT, HalfVT, V);
230  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
231  RoundParts / 2, PartVT, HalfVT, V);
232  } else {
233  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
234  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
235  }
236 
237  if (DAG.getDataLayout().isBigEndian())
238  std::swap(Lo, Hi);
239 
240  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
241 
242  if (RoundParts < NumParts) {
243  // Assemble the trailing non-power-of-2 part.
244  unsigned OddParts = NumParts - RoundParts;
245  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
246  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
247  OddVT, V, CC);
248 
249  // Combine the round and odd parts.
250  Lo = Val;
251  if (DAG.getDataLayout().isBigEndian())
252  std::swap(Lo, Hi);
253  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
254  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
255  Hi =
256  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
257  DAG.getConstant(Lo.getValueSizeInBits(), DL,
258  TLI.getPointerTy(DAG.getDataLayout())));
259  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
260  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
261  }
262  } else if (PartVT.isFloatingPoint()) {
263  // FP split into multiple FP parts (for ppcf128)
264  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
265  "Unexpected split");
266  SDValue Lo, Hi;
267  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
268  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
269  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
270  std::swap(Lo, Hi);
271  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
272  } else {
273  // FP split into integer parts (soft fp)
274  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
275  !PartVT.isVector() && "Unexpected split");
276  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
277  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
278  }
279  }
280 
281  // There is now one part, held in Val. Correct it to match ValueVT.
282  // PartEVT is the type of the register class that holds the value.
283  // ValueVT is the type of the inline asm operation.
284  EVT PartEVT = Val.getValueType();
285 
286  if (PartEVT == ValueVT)
287  return Val;
288 
289  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
290  ValueVT.bitsLT(PartEVT)) {
291  // For an FP value in an integer part, we need to truncate to the right
292  // width first.
293  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
294  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
295  }
296 
297  // Handle types that have the same size.
298  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
299  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
300 
301  // Handle types with different sizes.
302  if (PartEVT.isInteger() && ValueVT.isInteger()) {
303  if (ValueVT.bitsLT(PartEVT)) {
304  // For a truncate, see if we have any information to
305  // indicate whether the truncated bits will always be
306  // zero or sign-extension.
307  if (AssertOp.hasValue())
308  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
309  DAG.getValueType(ValueVT));
310  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
311  }
312  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
313  }
314 
315  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
316  // FP_ROUND's are always exact here.
317  if (ValueVT.bitsLT(Val.getValueType()))
318  return DAG.getNode(
319  ISD::FP_ROUND, DL, ValueVT, Val,
320  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
321 
322  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
323  }
324 
325  llvm_unreachable("Unknown mismatch!");
326 }
327 
329  const Twine &ErrMsg) {
330  const Instruction *I = dyn_cast_or_null<Instruction>(V);
331  if (!V)
332  return Ctx.emitError(ErrMsg);
333 
334  const char *AsmError = ", possible invalid constraint for vector type";
335  if (const CallInst *CI = dyn_cast<CallInst>(I))
336  if (isa<InlineAsm>(CI->getCalledValue()))
337  return Ctx.emitError(I, ErrMsg + AsmError);
338 
339  return Ctx.emitError(I, ErrMsg);
340 }
341 
342 /// getCopyFromPartsVector - Create a value that contains the specified legal
343 /// parts combined into the value they represent. If the parts combine to a
344 /// type larger than ValueVT then AssertOp can be used to specify whether the
345 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
346 /// ValueVT (ISD::AssertSext).
348  const SDValue *Parts, unsigned NumParts,
349  MVT PartVT, EVT ValueVT, const Value *V,
350  Optional<CallingConv::ID> CallConv) {
351  assert(ValueVT.isVector() && "Not a vector value");
352  assert(NumParts > 0 && "No parts to assemble!");
353  const bool IsABIRegCopy = CallConv.hasValue();
354 
355  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
356  SDValue Val = Parts[0];
357 
358  // Handle a multi-element vector.
359  if (NumParts > 1) {
360  EVT IntermediateVT;
361  MVT RegisterVT;
362  unsigned NumIntermediates;
363  unsigned NumRegs;
364 
365  if (IsABIRegCopy) {
367  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
368  NumIntermediates, RegisterVT);
369  } else {
370  NumRegs =
371  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
372  NumIntermediates, RegisterVT);
373  }
374 
375  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
376  NumParts = NumRegs; // Silence a compiler warning.
377  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
378  assert(RegisterVT.getSizeInBits() ==
379  Parts[0].getSimpleValueType().getSizeInBits() &&
380  "Part type sizes don't match!");
381 
382  // Assemble the parts into intermediate operands.
383  SmallVector<SDValue, 8> Ops(NumIntermediates);
384  if (NumIntermediates == NumParts) {
385  // If the register was not expanded, truncate or copy the value,
386  // as appropriate.
387  for (unsigned i = 0; i != NumParts; ++i)
388  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
389  PartVT, IntermediateVT, V);
390  } else if (NumParts > 0) {
391  // If the intermediate type was expanded, build the intermediate
392  // operands from the parts.
393  assert(NumParts % NumIntermediates == 0 &&
394  "Must expand into a divisible number of parts!");
395  unsigned Factor = NumParts / NumIntermediates;
396  for (unsigned i = 0; i != NumIntermediates; ++i)
397  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
398  PartVT, IntermediateVT, V);
399  }
400 
401  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
402  // intermediate operands.
403  EVT BuiltVectorTy =
404  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
405  (IntermediateVT.isVector()
406  ? IntermediateVT.getVectorNumElements() * NumParts
407  : NumIntermediates));
408  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
410  DL, BuiltVectorTy, Ops);
411  }
412 
413  // There is now one part, held in Val. Correct it to match ValueVT.
414  EVT PartEVT = Val.getValueType();
415 
416  if (PartEVT == ValueVT)
417  return Val;
418 
419  if (PartEVT.isVector()) {
420  // If the element type of the source/dest vectors are the same, but the
421  // parts vector has more elements than the value vector, then we have a
422  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
423  // elements we want.
424  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
425  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
426  "Cannot narrow, it would be a lossy transformation");
427  return DAG.getNode(
428  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
429  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
430  }
431 
432  // Vector/Vector bitcast.
433  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
434  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435 
436  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
437  "Cannot handle this kind of promotion");
438  // Promoted vector extract
439  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
440 
441  }
442 
443  // Trivial bitcast if the types are the same size and the destination
444  // vector type is legal.
445  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
446  TLI.isTypeLegal(ValueVT))
447  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
448 
449  if (ValueVT.getVectorNumElements() != 1) {
450  // Certain ABIs require that vectors are passed as integers. For vectors
451  // are the same size, this is an obvious bitcast.
452  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
453  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
454  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
455  // Bitcast Val back the original type and extract the corresponding
456  // vector we want.
457  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
458  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
459  ValueVT.getVectorElementType(), Elts);
460  Val = DAG.getBitcast(WiderVecType, Val);
461  return DAG.getNode(
462  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
463  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
464  }
465 
467  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
468  return DAG.getUNDEF(ValueVT);
469  }
470 
471  // Handle cases such as i8 -> <1 x i1>
472  EVT ValueSVT = ValueVT.getVectorElementType();
473  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
474  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
475  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
476 
477  return DAG.getBuildVector(ValueVT, DL, Val);
478 }
479 
480 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
481  SDValue Val, SDValue *Parts, unsigned NumParts,
482  MVT PartVT, const Value *V,
483  Optional<CallingConv::ID> CallConv);
484 
485 /// getCopyToParts - Create a series of nodes that contain the specified value
486 /// split into legal parts. If the parts contain more bits than Val, then, for
487 /// integers, ExtendKind can be used to specify how to generate the extra bits.
488 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
489  SDValue *Parts, unsigned NumParts, MVT PartVT,
490  const Value *V,
491  Optional<CallingConv::ID> CallConv = None,
492  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
493  EVT ValueVT = Val.getValueType();
494 
495  // Handle the vector case separately.
496  if (ValueVT.isVector())
497  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
498  CallConv);
499 
500  unsigned PartBits = PartVT.getSizeInBits();
501  unsigned OrigNumParts = NumParts;
502  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
503  "Copying to an illegal type!");
504 
505  if (NumParts == 0)
506  return;
507 
508  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
509  EVT PartEVT = PartVT;
510  if (PartEVT == ValueVT) {
511  assert(NumParts == 1 && "No-op copy with multiple parts!");
512  Parts[0] = Val;
513  return;
514  }
515 
516  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
517  // If the parts cover more bits than the value has, promote the value.
518  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
519  assert(NumParts == 1 && "Do not know what to promote to!");
520  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
521  } else {
522  if (ValueVT.isFloatingPoint()) {
523  // FP values need to be bitcast, then extended if they are being put
524  // into a larger container.
525  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
526  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
527  }
528  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529  ValueVT.isInteger() &&
530  "Unknown mismatch!");
531  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
533  if (PartVT == MVT::x86mmx)
534  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535  }
536  } else if (PartBits == ValueVT.getSizeInBits()) {
537  // Different types of the same size.
538  assert(NumParts == 1 && PartEVT != ValueVT);
539  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
540  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
541  // If the parts cover less bits than value has, truncate the value.
542  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
543  ValueVT.isInteger() &&
544  "Unknown mismatch!");
545  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
546  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
547  if (PartVT == MVT::x86mmx)
548  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
549  }
550 
551  // The value may have changed - recompute ValueVT.
552  ValueVT = Val.getValueType();
553  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
554  "Failed to tile the value with PartVT!");
555 
556  if (NumParts == 1) {
557  if (PartEVT != ValueVT) {
559  "scalar-to-vector conversion failed");
560  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561  }
562 
563  Parts[0] = Val;
564  return;
565  }
566 
567  // Expand the value into multiple parts.
568  if (NumParts & (NumParts - 1)) {
569  // The number of parts is not a power of 2. Split off and copy the tail.
570  assert(PartVT.isInteger() && ValueVT.isInteger() &&
571  "Do not know what to expand to!");
572  unsigned RoundParts = 1 << Log2_32(NumParts);
573  unsigned RoundBits = RoundParts * PartBits;
574  unsigned OddParts = NumParts - RoundParts;
575  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
576  DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
577 
578  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
579  CallConv);
580 
581  if (DAG.getDataLayout().isBigEndian())
582  // The odd parts were reversed by getCopyToParts - unreverse them.
583  std::reverse(Parts + RoundParts, Parts + NumParts);
584 
585  NumParts = RoundParts;
586  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
587  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
588  }
589 
590  // The number of parts is a power of 2. Repeatedly bisect the value using
591  // EXTRACT_ELEMENT.
592  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
594  ValueVT.getSizeInBits()),
595  Val);
596 
597  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
598  for (unsigned i = 0; i < NumParts; i += StepSize) {
599  unsigned ThisBits = StepSize * PartBits / 2;
600  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
601  SDValue &Part0 = Parts[i];
602  SDValue &Part1 = Parts[i+StepSize/2];
603 
604  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
605  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
606  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
607  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
608 
609  if (ThisBits == PartBits && ThisVT != PartVT) {
610  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
611  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
612  }
613  }
614  }
615 
616  if (DAG.getDataLayout().isBigEndian())
617  std::reverse(Parts, Parts + OrigNumParts);
618 }
619 
621  SDValue Val, const SDLoc &DL, EVT PartVT) {
622  if (!PartVT.isVector())
623  return SDValue();
624 
625  EVT ValueVT = Val.getValueType();
626  unsigned PartNumElts = PartVT.getVectorNumElements();
627  unsigned ValueNumElts = ValueVT.getVectorNumElements();
628  if (PartNumElts > ValueNumElts &&
629  PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
630  EVT ElementVT = PartVT.getVectorElementType();
631  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
632  // undef elements.
634  DAG.ExtractVectorElements(Val, Ops);
635  SDValue EltUndef = DAG.getUNDEF(ElementVT);
636  for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
637  Ops.push_back(EltUndef);
638 
639  // FIXME: Use CONCAT for 2x -> 4x.
640  return DAG.getBuildVector(PartVT, DL, Ops);
641  }
642 
643  return SDValue();
644 }
645 
646 /// getCopyToPartsVector - Create a series of nodes that contain the specified
647 /// value split into legal parts.
648 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
649  SDValue Val, SDValue *Parts, unsigned NumParts,
650  MVT PartVT, const Value *V,
651  Optional<CallingConv::ID> CallConv) {
652  EVT ValueVT = Val.getValueType();
653  assert(ValueVT.isVector() && "Not a vector");
654  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
655  const bool IsABIRegCopy = CallConv.hasValue();
656 
657  if (NumParts == 1) {
658  EVT PartEVT = PartVT;
659  if (PartEVT == ValueVT) {
660  // Nothing to do.
661  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
662  // Bitconvert vector->vector case.
663  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
664  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
665  Val = Widened;
666  } else if (PartVT.isVector() &&
667  PartEVT.getVectorElementType().bitsGE(
668  ValueVT.getVectorElementType()) &&
669  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
670 
671  // Promoted vector extract
672  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
673  } else {
674  if (ValueVT.getVectorNumElements() == 1) {
675  Val = DAG.getNode(
676  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
677  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
678  } else {
679  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
680  "lossy conversion of vector to scalar type");
681  EVT IntermediateType =
682  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
683  Val = DAG.getBitcast(IntermediateType, Val);
684  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
685  }
686  }
687 
688  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
689  Parts[0] = Val;
690  return;
691  }
692 
693  // Handle a multi-element vector.
694  EVT IntermediateVT;
695  MVT RegisterVT;
696  unsigned NumIntermediates;
697  unsigned NumRegs;
698  if (IsABIRegCopy) {
699  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
700  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
701  NumIntermediates, RegisterVT);
702  } else {
703  NumRegs =
704  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
705  NumIntermediates, RegisterVT);
706  }
707 
708  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
709  NumParts = NumRegs; // Silence a compiler warning.
710  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
711 
712  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
713  IntermediateVT.getVectorNumElements() : 1;
714 
715  // Convert the vector to the appropiate type if necessary.
716  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
717 
718  EVT BuiltVectorTy = EVT::getVectorVT(
719  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
720  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
721  if (ValueVT != BuiltVectorTy) {
722  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
723  Val = Widened;
724 
725  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
726  }
727 
728  // Split the vector into intermediate operands.
729  SmallVector<SDValue, 8> Ops(NumIntermediates);
730  for (unsigned i = 0; i != NumIntermediates; ++i) {
731  if (IntermediateVT.isVector()) {
732  Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
733  DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
734  } else {
735  Ops[i] = DAG.getNode(
736  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
737  DAG.getConstant(i, DL, IdxVT));
738  }
739  }
740 
741  // Split the intermediate operands into legal parts.
742  if (NumParts == NumIntermediates) {
743  // If the register was not expanded, promote or copy the value,
744  // as appropriate.
745  for (unsigned i = 0; i != NumParts; ++i)
746  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
747  } else if (NumParts > 0) {
748  // If the intermediate type was expanded, split each the value into
749  // legal parts.
750  assert(NumIntermediates != 0 && "division by zero");
751  assert(NumParts % NumIntermediates == 0 &&
752  "Must expand into a divisible number of parts!");
753  unsigned Factor = NumParts / NumIntermediates;
754  for (unsigned i = 0; i != NumIntermediates; ++i)
755  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
756  CallConv);
757  }
758 }
759 
761  EVT valuevt, Optional<CallingConv::ID> CC)
762  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
763  RegCount(1, regs.size()), CallConv(CC) {}
764 
766  const DataLayout &DL, unsigned Reg, Type *Ty,
768  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
769 
770  CallConv = CC;
771 
772  for (EVT ValueVT : ValueVTs) {
773  unsigned NumRegs =
774  isABIMangled()
775  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
776  : TLI.getNumRegisters(Context, ValueVT);
777  MVT RegisterVT =
778  isABIMangled()
779  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
780  : TLI.getRegisterType(Context, ValueVT);
781  for (unsigned i = 0; i != NumRegs; ++i)
782  Regs.push_back(Reg + i);
783  RegVTs.push_back(RegisterVT);
784  RegCount.push_back(NumRegs);
785  Reg += NumRegs;
786  }
787 }
788 
790  FunctionLoweringInfo &FuncInfo,
791  const SDLoc &dl, SDValue &Chain,
792  SDValue *Flag, const Value *V) const {
793  // A Value with type {} or [0 x %t] needs no registers.
794  if (ValueVTs.empty())
795  return SDValue();
796 
797  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
798 
799  // Assemble the legal parts into the final values.
800  SmallVector<SDValue, 4> Values(ValueVTs.size());
802  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
803  // Copy the legal parts from the registers.
804  EVT ValueVT = ValueVTs[Value];
805  unsigned NumRegs = RegCount[Value];
806  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
807  *DAG.getContext(),
808  CallConv.getValue(), RegVTs[Value])
809  : RegVTs[Value];
810 
811  Parts.resize(NumRegs);
812  for (unsigned i = 0; i != NumRegs; ++i) {
813  SDValue P;
814  if (!Flag) {
815  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
816  } else {
817  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
818  *Flag = P.getValue(2);
819  }
820 
821  Chain = P.getValue(1);
822  Parts[i] = P;
823 
824  // If the source register was virtual and if we know something about it,
825  // add an assert node.
827  !RegisterVT.isInteger())
828  continue;
829 
831  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
832  if (!LOI)
833  continue;
834 
835  unsigned RegSize = RegisterVT.getScalarSizeInBits();
836  unsigned NumSignBits = LOI->NumSignBits;
837  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
838 
839  if (NumZeroBits == RegSize) {
840  // The current value is a zero.
841  // Explicitly express that as it would be easier for
842  // optimizations to kick in.
843  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
844  continue;
845  }
846 
847  // FIXME: We capture more information than the dag can represent. For
848  // now, just use the tightest assertzext/assertsext possible.
849  bool isSExt;
850  EVT FromVT(MVT::Other);
851  if (NumZeroBits) {
852  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
853  isSExt = false;
854  } else if (NumSignBits > 1) {
855  FromVT =
856  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
857  isSExt = true;
858  } else {
859  continue;
860  }
861  // Add an assertion node.
862  assert(FromVT != MVT::Other);
863  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
864  RegisterVT, P, DAG.getValueType(FromVT));
865  }
866 
867  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
868  RegisterVT, ValueVT, V, CallConv);
869  Part += NumRegs;
870  Parts.clear();
871  }
872 
873  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
874 }
875 
877  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
878  const Value *V,
879  ISD::NodeType PreferredExtendType) const {
880  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
881  ISD::NodeType ExtendKind = PreferredExtendType;
882 
883  // Get the list of the values's legal parts.
884  unsigned NumRegs = Regs.size();
885  SmallVector<SDValue, 8> Parts(NumRegs);
886  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
887  unsigned NumParts = RegCount[Value];
888 
889  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
890  *DAG.getContext(),
891  CallConv.getValue(), RegVTs[Value])
892  : RegVTs[Value];
893 
894  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
895  ExtendKind = ISD::ZERO_EXTEND;
896 
897  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
898  NumParts, RegisterVT, V, CallConv, ExtendKind);
899  Part += NumParts;
900  }
901 
902  // Copy the parts into the registers.
903  SmallVector<SDValue, 8> Chains(NumRegs);
904  for (unsigned i = 0; i != NumRegs; ++i) {
905  SDValue Part;
906  if (!Flag) {
907  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
908  } else {
909  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
910  *Flag = Part.getValue(1);
911  }
912 
913  Chains[i] = Part.getValue(0);
914  }
915 
916  if (NumRegs == 1 || Flag)
917  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
918  // flagged to it. That is the CopyToReg nodes and the user are considered
919  // a single scheduling unit. If we create a TokenFactor and return it as
920  // chain, then the TokenFactor is both a predecessor (operand) of the
921  // user as well as a successor (the TF operands are flagged to the user).
922  // c1, f1 = CopyToReg
923  // c2, f2 = CopyToReg
924  // c3 = TokenFactor c1, c2
925  // ...
926  // = op c3, ..., f2
927  Chain = Chains[NumRegs-1];
928  else
929  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
930 }
931 
932 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
933  unsigned MatchingIdx, const SDLoc &dl,
934  SelectionDAG &DAG,
935  std::vector<SDValue> &Ops) const {
936  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
937 
938  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
939  if (HasMatching)
940  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
941  else if (!Regs.empty() &&
943  // Put the register class of the virtual registers in the flag word. That
944  // way, later passes can recompute register class constraints for inline
945  // assembly as well as normal instructions.
946  // Don't do this for tied operands that can use the regclass information
947  // from the def.
949  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
950  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
951  }
952 
953  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
954  Ops.push_back(Res);
955 
956  if (Code == InlineAsm::Kind_Clobber) {
957  // Clobbers should always have a 1:1 mapping with registers, and may
958  // reference registers that have illegal (e.g. vector) types. Hence, we
959  // shouldn't try to apply any sort of splitting logic to them.
960  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
961  "No 1:1 mapping from clobbers to regs?");
962  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
963  (void)SP;
964  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
965  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
966  assert(
967  (Regs[I] != SP ||
969  "If we clobbered the stack pointer, MFI should know about it.");
970  }
971  return;
972  }
973 
974  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
975  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
976  MVT RegisterVT = RegVTs[Value];
977  for (unsigned i = 0; i != NumRegs; ++i) {
978  assert(Reg < Regs.size() && "Mismatch in # registers expected");
979  unsigned TheReg = Regs[Reg++];
980  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
981  }
982  }
983 }
984 
988  unsigned I = 0;
989  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
990  unsigned RegCount = std::get<0>(CountAndVT);
991  MVT RegisterVT = std::get<1>(CountAndVT);
992  unsigned RegisterSize = RegisterVT.getSizeInBits();
993  for (unsigned E = I + RegCount; I != E; ++I)
994  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
995  }
996  return OutVec;
997 }
998 
1000  const TargetLibraryInfo *li) {
1001  AA = aa;
1002  GFI = gfi;
1003  LibInfo = li;
1004  DL = &DAG.getDataLayout();
1005  Context = DAG.getContext();
1006  LPadToCallSiteMap.clear();
1007 }
1008 
1010  NodeMap.clear();
1011  UnusedArgNodeMap.clear();
1012  PendingLoads.clear();
1013  PendingExports.clear();
1014  CurInst = nullptr;
1015  HasTailCall = false;
1016  SDNodeOrder = LowestSDNodeOrder;
1017  StatepointLowering.clear();
1018 }
1019 
1021  DanglingDebugInfoMap.clear();
1022 }
1023 
1025  if (PendingLoads.empty())
1026  return DAG.getRoot();
1027 
1028  if (PendingLoads.size() == 1) {
1029  SDValue Root = PendingLoads[0];
1030  DAG.setRoot(Root);
1031  PendingLoads.clear();
1032  return Root;
1033  }
1034 
1035  // Otherwise, we have to make a token factor node.
1036  SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1037  PendingLoads.clear();
1038  DAG.setRoot(Root);
1039  return Root;
1040 }
1041 
1043  SDValue Root = DAG.getRoot();
1044 
1045  if (PendingExports.empty())
1046  return Root;
1047 
1048  // Turn all of the CopyToReg chains into one factored node.
1049  if (Root.getOpcode() != ISD::EntryToken) {
1050  unsigned i = 0, e = PendingExports.size();
1051  for (; i != e; ++i) {
1052  assert(PendingExports[i].getNode()->getNumOperands() > 1);
1053  if (PendingExports[i].getNode()->getOperand(0) == Root)
1054  break; // Don't add the root if we already indirectly depend on it.
1055  }
1056 
1057  if (i == e)
1058  PendingExports.push_back(Root);
1059  }
1060 
1061  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1062  PendingExports);
1063  PendingExports.clear();
1064  DAG.setRoot(Root);
1065  return Root;
1066 }
1067 
1069  // Set up outgoing PHI node register values before emitting the terminator.
1070  if (I.isTerminator()) {
1071  HandlePHINodesInSuccessorBlocks(I.getParent());
1072  }
1073 
1074  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1075  if (!isa<DbgInfoIntrinsic>(I))
1076  ++SDNodeOrder;
1077 
1078  CurInst = &I;
1079 
1080  visit(I.getOpcode(), I);
1081 
1082  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1083  // Propagate the fast-math-flags of this IR instruction to the DAG node that
1084  // maps to this instruction.
1085  // TODO: We could handle all flags (nsw, etc) here.
1086  // TODO: If an IR instruction maps to >1 node, only the final node will have
1087  // flags set.
1088  if (SDNode *Node = getNodeForIRValue(&I)) {
1089  SDNodeFlags IncomingFlags;
1090  IncomingFlags.copyFMF(*FPMO);
1091  if (!Node->getFlags().isDefined())
1092  Node->setFlags(IncomingFlags);
1093  else
1094  Node->intersectFlagsWith(IncomingFlags);
1095  }
1096  }
1097 
1098  if (!I.isTerminator() && !HasTailCall &&
1099  !isStatepoint(&I)) // statepoints handle their exports internally
1100  CopyToExportRegsIfNeeded(&I);
1101 
1102  CurInst = nullptr;
1103 }
1104 
1105 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1106  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1107 }
1108 
1109 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1110  // Note: this doesn't use InstVisitor, because it has to work with
1111  // ConstantExpr's in addition to instructions.
1112  switch (Opcode) {
1113  default: llvm_unreachable("Unknown instruction type encountered!");
1114  // Build the switch statement using the Instruction.def file.
1115 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1116  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1117 #include "llvm/IR/Instruction.def"
1118  }
1119 }
1120 
1122  const DIExpression *Expr) {
1123  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1124  const DbgValueInst *DI = DDI.getDI();
1125  DIVariable *DanglingVariable = DI->getVariable();
1126  DIExpression *DanglingExpr = DI->getExpression();
1127  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1128  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1129  return true;
1130  }
1131  return false;
1132  };
1133 
1134  for (auto &DDIMI : DanglingDebugInfoMap) {
1135  DanglingDebugInfoVector &DDIV = DDIMI.second;
1136 
1137  // If debug info is to be dropped, run it through final checks to see
1138  // whether it can be salvaged.
1139  for (auto &DDI : DDIV)
1140  if (isMatchingDbgValue(DDI))
1141  salvageUnresolvedDbgValue(DDI);
1142 
1143  DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1144  }
1145 }
1146 
1147 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1148 // generate the debug data structures now that we've seen its definition.
1150  SDValue Val) {
1151  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1152  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1153  return;
1154 
1155  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1156  for (auto &DDI : DDIV) {
1157  const DbgValueInst *DI = DDI.getDI();
1158  assert(DI && "Ill-formed DanglingDebugInfo");
1159  DebugLoc dl = DDI.getdl();
1160  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1161  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1162  DILocalVariable *Variable = DI->getVariable();
1163  DIExpression *Expr = DI->getExpression();
1164  assert(Variable->isValidLocationForIntrinsic(dl) &&
1165  "Expected inlined-at fields to agree");
1166  SDDbgValue *SDV;
1167  if (Val.getNode()) {
1168  // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1169  // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1170  // we couldn't resolve it directly when examining the DbgValue intrinsic
1171  // in the first place we should not be more successful here). Unless we
1172  // have some test case that prove this to be correct we should avoid
1173  // calling EmitFuncArgumentDbgValue here.
1174  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1175  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1176  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1177  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1178  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1179  // inserted after the definition of Val when emitting the instructions
1180  // after ISel. An alternative could be to teach
1181  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1182  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1183  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1184  << ValSDNodeOrder << "\n");
1185  SDV = getDbgValue(Val, Variable, Expr, dl,
1186  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1187  DAG.AddDbgValue(SDV, Val.getNode(), false);
1188  } else
1189  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1190  << "in EmitFuncArgumentDbgValue\n");
1191  } else {
1192  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1193  auto Undef =
1194  UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1195  auto SDV =
1196  DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1197  DAG.AddDbgValue(SDV, nullptr, false);
1198  }
1199  }
1200  DDIV.clear();
1201 }
1202 
1204  Value *V = DDI.getDI()->getValue();
1205  DILocalVariable *Var = DDI.getDI()->getVariable();
1206  DIExpression *Expr = DDI.getDI()->getExpression();
1207  DebugLoc DL = DDI.getdl();
1208  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1209  unsigned SDOrder = DDI.getSDNodeOrder();
1210 
1211  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1212  // that DW_OP_stack_value is desired.
1213  assert(isa<DbgValueInst>(DDI.getDI()));
1214  bool StackValue = true;
1215 
1216  // Can this Value can be encoded without any further work?
1217  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1218  return;
1219 
1220  // Attempt to salvage back through as many instructions as possible. Bail if
1221  // a non-instruction is seen, such as a constant expression or global
1222  // variable. FIXME: Further work could recover those too.
1223  while (isa<Instruction>(V)) {
1224  Instruction &VAsInst = *cast<Instruction>(V);
1225  DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
1226 
1227  // If we cannot salvage any further, and haven't yet found a suitable debug
1228  // expression, bail out.
1229  if (!NewExpr)
1230  break;
1231 
1232  // New value and expr now represent this debuginfo.
1233  V = VAsInst.getOperand(0);
1234  Expr = NewExpr;
1235 
1236  // Some kind of simplification occurred: check whether the operand of the
1237  // salvaged debug expression can be encoded in this DAG.
1238  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1239  LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "
1240  << DDI.getDI() << "\nBy stripping back to:\n " << V);
1241  return;
1242  }
1243  }
1244 
1245  // This was the final opportunity to salvage this debug information, and it
1246  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1247  // any earlier variable location.
1248  auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1249  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1250  DAG.AddDbgValue(SDV, nullptr, false);
1251 
1252  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()
1253  << "\n");
1254  LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)
1255  << "\n");
1256 }
1257 
1259  DIExpression *Expr, DebugLoc dl,
1260  DebugLoc InstDL, unsigned Order) {
1261  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1262  SDDbgValue *SDV;
1263  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1264  isa<ConstantPointerNull>(V)) {
1265  SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
1266  DAG.AddDbgValue(SDV, nullptr, false);
1267  return true;
1268  }
1269 
1270  // If the Value is a frame index, we can create a FrameIndex debug value
1271  // without relying on the DAG at all.
1272  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1273  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1274  if (SI != FuncInfo.StaticAllocaMap.end()) {
1275  auto SDV =
1276  DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
1277  /*IsIndirect*/ false, dl, SDNodeOrder);
1278  // Do not attach the SDNodeDbgValue to an SDNode: this variable location
1279  // is still available even if the SDNode gets optimized out.
1280  DAG.AddDbgValue(SDV, nullptr, false);
1281  return true;
1282  }
1283  }
1284 
1285  // Do not use getValue() in here; we don't want to generate code at
1286  // this point if it hasn't been done yet.
1287  SDValue N = NodeMap[V];
1288  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1289  N = UnusedArgNodeMap[V];
1290  if (N.getNode()) {
1291  if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1292  return true;
1293  SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1294  DAG.AddDbgValue(SDV, N.getNode(), false);
1295  return true;
1296  }
1297 
1298  // Special rules apply for the first dbg.values of parameter variables in a
1299  // function. Identify them by the fact they reference Argument Values, that
1300  // they're parameters, and they are parameters of the current function. We
1301  // need to let them dangle until they get an SDNode.
1302  bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&
1303  !InstDL.getInlinedAt();
1304  if (!IsParamOfFunc) {
1305  // The value is not used in this block yet (or it would have an SDNode).
1306  // We still want the value to appear for the user if possible -- if it has
1307  // an associated VReg, we can refer to that instead.
1308  auto VMI = FuncInfo.ValueMap.find(V);
1309  if (VMI != FuncInfo.ValueMap.end()) {
1310  unsigned Reg = VMI->second;
1311  // If this is a PHI node, it may be split up into several MI PHI nodes
1312  // (in FunctionLoweringInfo::set).
1313  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1314  V->getType(), None);
1315  if (RFV.occupiesMultipleRegs()) {
1316  unsigned Offset = 0;
1317  unsigned BitsToDescribe = 0;
1318  if (auto VarSize = Var->getSizeInBits())
1319  BitsToDescribe = *VarSize;
1320  if (auto Fragment = Expr->getFragmentInfo())
1321  BitsToDescribe = Fragment->SizeInBits;
1322  for (auto RegAndSize : RFV.getRegsAndSizes()) {
1323  unsigned RegisterSize = RegAndSize.second;
1324  // Bail out if all bits are described already.
1325  if (Offset >= BitsToDescribe)
1326  break;
1327  unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1328  ? BitsToDescribe - Offset
1329  : RegisterSize;
1330  auto FragmentExpr = DIExpression::createFragmentExpression(
1331  Expr, Offset, FragmentSize);
1332  if (!FragmentExpr)
1333  continue;
1334  SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
1335  false, dl, SDNodeOrder);
1336  DAG.AddDbgValue(SDV, nullptr, false);
1337  Offset += RegisterSize;
1338  }
1339  } else {
1340  SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
1341  DAG.AddDbgValue(SDV, nullptr, false);
1342  }
1343  return true;
1344  }
1345  }
1346 
1347  return false;
1348 }
1349 
1351  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1352  for (auto &Pair : DanglingDebugInfoMap)
1353  for (auto &DDI : Pair.getSecond())
1354  salvageUnresolvedDbgValue(DDI);
1355  clearDanglingDebugInfo();
1356 }
1357 
1358 /// getCopyFromRegs - If there was virtual register allocated for the value V
1359 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1361  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1362  SDValue Result;
1363 
1364  if (It != FuncInfo.ValueMap.end()) {
1365  unsigned InReg = It->second;
1366 
1367  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1368  DAG.getDataLayout(), InReg, Ty,
1369  None); // This is not an ABI copy.
1370  SDValue Chain = DAG.getEntryNode();
1371  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1372  V);
1373  resolveDanglingDebugInfo(V, Result);
1374  }
1375 
1376  return Result;
1377 }
1378 
1379 /// getValue - Return an SDValue for the given Value.
1381  // If we already have an SDValue for this value, use it. It's important
1382  // to do this first, so that we don't create a CopyFromReg if we already
1383  // have a regular SDValue.
1384  SDValue &N = NodeMap[V];
1385  if (N.getNode()) return N;
1386 
1387  // If there's a virtual register allocated and initialized for this
1388  // value, use it.
1389  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1390  return copyFromReg;
1391 
1392  // Otherwise create a new SDValue and remember it.
1393  SDValue Val = getValueImpl(V);
1394  NodeMap[V] = Val;
1395  resolveDanglingDebugInfo(V, Val);
1396  return Val;
1397 }
1398 
1399 // Return true if SDValue exists for the given Value
1401  return (NodeMap.find(V) != NodeMap.end()) ||
1402  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1403 }
1404 
1405 /// getNonRegisterValue - Return an SDValue for the given Value, but
1406 /// don't look in FuncInfo.ValueMap for a virtual register.
1408  // If we already have an SDValue for this value, use it.
1409  SDValue &N = NodeMap[V];
1410  if (N.getNode()) {
1411  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1412  // Remove the debug location from the node as the node is about to be used
1413  // in a location which may differ from the original debug location. This
1414  // is relevant to Constant and ConstantFP nodes because they can appear
1415  // as constant expressions inside PHI nodes.
1416  N->setDebugLoc(DebugLoc());
1417  }
1418  return N;
1419  }
1420 
1421  // Otherwise create a new SDValue and remember it.
1422  SDValue Val = getValueImpl(V);
1423  NodeMap[V] = Val;
1424  resolveDanglingDebugInfo(V, Val);
1425  return Val;
1426 }
1427 
1428 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1429 /// Create an SDValue for the given value.
1431  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1432 
1433  if (const Constant *C = dyn_cast<Constant>(V)) {
1434  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1435 
1436  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1437  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1438 
1439  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1440  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1441 
1442  if (isa<ConstantPointerNull>(C)) {
1443  unsigned AS = V->getType()->getPointerAddressSpace();
1444  return DAG.getConstant(0, getCurSDLoc(),
1445  TLI.getPointerTy(DAG.getDataLayout(), AS));
1446  }
1447 
1448  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1449  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1450 
1451  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1452  return DAG.getUNDEF(VT);
1453 
1454  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1455  visit(CE->getOpcode(), *CE);
1456  SDValue N1 = NodeMap[V];
1457  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1458  return N1;
1459  }
1460 
1461  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1463  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1464  OI != OE; ++OI) {
1465  SDNode *Val = getValue(*OI).getNode();
1466  // If the operand is an empty aggregate, there are no values.
1467  if (!Val) continue;
1468  // Add each leaf value from the operand to the Constants list
1469  // to form a flattened list of all the values.
1470  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1471  Constants.push_back(SDValue(Val, i));
1472  }
1473 
1474  return DAG.getMergeValues(Constants, getCurSDLoc());
1475  }
1476 
1477  if (const ConstantDataSequential *CDS =
1478  dyn_cast<ConstantDataSequential>(C)) {
1480  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1481  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1482  // Add each leaf value from the operand to the Constants list
1483  // to form a flattened list of all the values.
1484  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1485  Ops.push_back(SDValue(Val, i));
1486  }
1487 
1488  if (isa<ArrayType>(CDS->getType()))
1489  return DAG.getMergeValues(Ops, getCurSDLoc());
1490  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1491  }
1492 
1493  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1494  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1495  "Unknown struct or array constant!");
1496 
1498  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1499  unsigned NumElts = ValueVTs.size();
1500  if (NumElts == 0)
1501  return SDValue(); // empty struct
1503  for (unsigned i = 0; i != NumElts; ++i) {
1504  EVT EltVT = ValueVTs[i];
1505  if (isa<UndefValue>(C))
1506  Constants[i] = DAG.getUNDEF(EltVT);
1507  else if (EltVT.isFloatingPoint())
1508  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1509  else
1510  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1511  }
1512 
1513  return DAG.getMergeValues(Constants, getCurSDLoc());
1514  }
1515 
1516  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1517  return DAG.getBlockAddress(BA, VT);
1518 
1519  VectorType *VecTy = cast<VectorType>(V->getType());
1520  unsigned NumElements = VecTy->getNumElements();
1521 
1522  // Now that we know the number and type of the elements, get that number of
1523  // elements into the Ops array based on what kind of constant it is.
1525  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1526  for (unsigned i = 0; i != NumElements; ++i)
1527  Ops.push_back(getValue(CV->getOperand(i)));
1528  } else {
1529  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1530  EVT EltVT =
1531  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1532 
1533  SDValue Op;
1534  if (EltVT.isFloatingPoint())
1535  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1536  else
1537  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1538  Ops.assign(NumElements, Op);
1539  }
1540 
1541  // Create a BUILD_VECTOR node.
1542  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1543  }
1544 
1545  // If this is a static alloca, generate it as the frameindex instead of
1546  // computation.
1547  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1549  FuncInfo.StaticAllocaMap.find(AI);
1550  if (SI != FuncInfo.StaticAllocaMap.end())
1551  return DAG.getFrameIndex(SI->second,
1552  TLI.getFrameIndexTy(DAG.getDataLayout()));
1553  }
1554 
1555  // If this is an instruction which fast-isel has deferred, select it now.
1556  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1557  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1558 
1559  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1560  Inst->getType(), getABIRegCopyCC(V));
1561  SDValue Chain = DAG.getEntryNode();
1562  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1563  }
1564 
1565  llvm_unreachable("Can't get register for value!");
1566 }
1567 
1568 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1569  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1570  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1571  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1572  bool IsSEH = isAsynchronousEHPersonality(Pers);
1573  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1574  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1575  if (!IsSEH)
1576  CatchPadMBB->setIsEHScopeEntry();
1577  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1578  if (IsMSVCCXX || IsCoreCLR)
1579  CatchPadMBB->setIsEHFuncletEntry();
1580  // Wasm does not need catchpads anymore
1581  if (!IsWasmCXX)
1582  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1583  getControlRoot()));
1584 }
1585 
1586 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1587  // Update machine-CFG edge.
1588  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1589  FuncInfo.MBB->addSuccessor(TargetMBB);
1590 
1591  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1592  bool IsSEH = isAsynchronousEHPersonality(Pers);
1593  if (IsSEH) {
1594  // If this is not a fall-through branch or optimizations are switched off,
1595  // emit the branch.
1596  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1597  TM.getOptLevel() == CodeGenOpt::None)
1598  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1599  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1600  return;
1601  }
1602 
1603  // Figure out the funclet membership for the catchret's successor.
1604  // This will be used by the FuncletLayout pass to determine how to order the
1605  // BB's.
1606  // A 'catchret' returns to the outer scope's color.
1607  Value *ParentPad = I.getCatchSwitchParentPad();
1608  const BasicBlock *SuccessorColor;
1609  if (isa<ConstantTokenNone>(ParentPad))
1610  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1611  else
1612  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1613  assert(SuccessorColor && "No parent funclet for catchret!");
1614  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1615  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1616 
1617  // Create the terminator node.
1618  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1619  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1620  DAG.getBasicBlock(SuccessorColorMBB));
1621  DAG.setRoot(Ret);
1622 }
1623 
1624 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1625  // Don't emit any special code for the cleanuppad instruction. It just marks
1626  // the start of an EH scope/funclet.
1627  FuncInfo.MBB->setIsEHScopeEntry();
1628  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1629  if (Pers != EHPersonality::Wasm_CXX) {
1630  FuncInfo.MBB->setIsEHFuncletEntry();
1631  FuncInfo.MBB->setIsCleanupFuncletEntry();
1632  }
1633 }
1634 
1635 // For wasm, there's alwyas a single catch pad attached to a catchswitch, and
1636 // the control flow always stops at the single catch pad, as it does for a
1637 // cleanup pad. In case the exception caught is not of the types the catch pad
1638 // catches, it will be rethrown by a rethrow.
1640  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1641  BranchProbability Prob,
1642  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1643  &UnwindDests) {
1644  while (EHPadBB) {
1645  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1646  if (isa<CleanupPadInst>(Pad)) {
1647  // Stop on cleanup pads.
1648  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1649  UnwindDests.back().first->setIsEHScopeEntry();
1650  break;
1651  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1652  // Add the catchpad handlers to the possible destinations. We don't
1653  // continue to the unwind destination of the catchswitch for wasm.
1654  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1655  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1656  UnwindDests.back().first->setIsEHScopeEntry();
1657  }
1658  break;
1659  } else {
1660  continue;
1661  }
1662  }
1663 }
1664 
1665 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1666 /// many places it could ultimately go. In the IR, we have a single unwind
1667 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1668 /// This function skips over imaginary basic blocks that hold catchswitch
1669 /// instructions, and finds all the "real" machine
1670 /// basic block destinations. As those destinations may not be successors of
1671 /// EHPadBB, here we also calculate the edge probability to those destinations.
1672 /// The passed-in Prob is the edge probability to EHPadBB.
1674  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1675  BranchProbability Prob,
1676  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1677  &UnwindDests) {
1678  EHPersonality Personality =
1680  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1681  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1682  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1683  bool IsSEH = isAsynchronousEHPersonality(Personality);
1684 
1685  if (IsWasmCXX) {
1686  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1687  return;
1688  }
1689 
1690  while (EHPadBB) {
1691  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1692  BasicBlock *NewEHPadBB = nullptr;
1693  if (isa<LandingPadInst>(Pad)) {
1694  // Stop on landingpads. They are not funclets.
1695  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1696  break;
1697  } else if (isa<CleanupPadInst>(Pad)) {
1698  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1699  // personalities.
1700  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1701  UnwindDests.back().first->setIsEHScopeEntry();
1702  UnwindDests.back().first->setIsEHFuncletEntry();
1703  break;
1704  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1705  // Add the catchpad handlers to the possible destinations.
1706  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1707  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1708  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1709  if (IsMSVCCXX || IsCoreCLR)
1710  UnwindDests.back().first->setIsEHFuncletEntry();
1711  if (!IsSEH)
1712  UnwindDests.back().first->setIsEHScopeEntry();
1713  }
1714  NewEHPadBB = CatchSwitch->getUnwindDest();
1715  } else {
1716  continue;
1717  }
1718 
1719  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1720  if (BPI && NewEHPadBB)
1721  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1722  EHPadBB = NewEHPadBB;
1723  }
1724 }
1725 
1726 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1727  // Update successor info.
1729  auto UnwindDest = I.getUnwindDest();
1730  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1731  BranchProbability UnwindDestProb =
1732  (BPI && UnwindDest)
1733  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1735  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1736  for (auto &UnwindDest : UnwindDests) {
1737  UnwindDest.first->setIsEHPad();
1738  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1739  }
1740  FuncInfo.MBB->normalizeSuccProbs();
1741 
1742  // Create the terminator node.
1743  SDValue Ret =
1744  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1745  DAG.setRoot(Ret);
1746 }
1747 
1748 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1749  report_fatal_error("visitCatchSwitch not yet implemented!");
1750 }
1751 
1752 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1753  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1754  auto &DL = DAG.getDataLayout();
1755  SDValue Chain = getControlRoot();
1757  SmallVector<SDValue, 8> OutVals;
1758 
1759  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1760  // lower
1761  //
1762  // %val = call <ty> @llvm.experimental.deoptimize()
1763  // ret <ty> %val
1764  //
1765  // differently.
1767  LowerDeoptimizingReturn();
1768  return;
1769  }
1770 
1771  if (!FuncInfo.CanLowerReturn) {
1772  unsigned DemoteReg = FuncInfo.DemoteRegister;
1773  const Function *F = I.getParent()->getParent();
1774 
1775  // Emit a store of the return value through the virtual register.
1776  // Leave Outs empty so that LowerReturn won't try to load return
1777  // registers the usual way.
1778  SmallVector<EVT, 1> PtrValueVTs;
1779  ComputeValueVTs(TLI, DL,
1782  PtrValueVTs);
1783 
1784  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1785  DemoteReg, PtrValueVTs[0]);
1786  SDValue RetOp = getValue(I.getOperand(0));
1787 
1790  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1791  unsigned NumValues = ValueVTs.size();
1792 
1793  SmallVector<SDValue, 4> Chains(NumValues);
1794  for (unsigned i = 0; i != NumValues; ++i) {
1795  // An aggregate return value cannot wrap around the address space, so
1796  // offsets to its parts don't wrap either.
1797  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1798  Chains[i] = DAG.getStore(
1799  Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1800  // FIXME: better loc info would be nice.
1802  }
1803 
1804  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1805  MVT::Other, Chains);
1806  } else if (I.getNumOperands() != 0) {
1808  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1809  unsigned NumValues = ValueVTs.size();
1810  if (NumValues) {
1811  SDValue RetOp = getValue(I.getOperand(0));
1812 
1813  const Function *F = I.getParent()->getParent();
1814 
1815  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1816  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1817  Attribute::SExt))
1818  ExtendKind = ISD::SIGN_EXTEND;
1819  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1820  Attribute::ZExt))
1821  ExtendKind = ISD::ZERO_EXTEND;
1822 
1823  LLVMContext &Context = F->getContext();
1824  bool RetInReg = F->getAttributes().hasAttribute(
1825  AttributeList::ReturnIndex, Attribute::InReg);
1826 
1827  for (unsigned j = 0; j != NumValues; ++j) {
1828  EVT VT = ValueVTs[j];
1829 
1830  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1831  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1832 
1833  CallingConv::ID CC = F->getCallingConv();
1834 
1835  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1836  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1837  SmallVector<SDValue, 4> Parts(NumParts);
1838  getCopyToParts(DAG, getCurSDLoc(),
1839  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1840  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1841 
1842  // 'inreg' on function refers to return value
1843  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1844  if (RetInReg)
1845  Flags.setInReg();
1846 
1847  // Propagate extension type if any
1848  if (ExtendKind == ISD::SIGN_EXTEND)
1849  Flags.setSExt();
1850  else if (ExtendKind == ISD::ZERO_EXTEND)
1851  Flags.setZExt();
1852 
1853  for (unsigned i = 0; i < NumParts; ++i) {
1854  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1855  VT, /*isfixed=*/true, 0, 0));
1856  OutVals.push_back(Parts[i]);
1857  }
1858  }
1859  }
1860  }
1861 
1862  // Push in swifterror virtual register as the last element of Outs. This makes
1863  // sure swifterror virtual register will be returned in the swifterror
1864  // physical register.
1865  const Function *F = I.getParent()->getParent();
1866  if (TLI.supportSwiftError() &&
1867  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1868  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1869  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1870  Flags.setSwiftError();
1871  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1872  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1873  true /*isfixed*/, 1 /*origidx*/,
1874  0 /*partOffs*/));
1875  // Create SDNode for the swifterror virtual register.
1876  OutVals.push_back(
1877  DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1878  &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1879  EVT(TLI.getPointerTy(DL))));
1880  }
1881 
1882  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1883  CallingConv::ID CallConv =
1885  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1886  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1887 
1888  // Verify that the target's LowerReturn behaved as expected.
1889  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1890  "LowerReturn didn't return a valid chain!");
1891 
1892  // Update the DAG with the new chain value resulting from return lowering.
1893  DAG.setRoot(Chain);
1894 }
1895 
1896 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1897 /// created for it, emit nodes to copy the value into the virtual
1898 /// registers.
1900  // Skip empty types
1901  if (V->getType()->isEmptyTy())
1902  return;
1903 
1904  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1905  if (VMI != FuncInfo.ValueMap.end()) {
1906  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1907  CopyValueToVirtualRegister(V, VMI->second);
1908  }
1909 }
1910 
1911 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1912 /// the current basic block, add it to ValueMap now so that we'll get a
1913 /// CopyTo/FromReg.
1915  // No need to export constants.
1916  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1917 
1918  // Already exported?
1919  if (FuncInfo.isExportedInst(V)) return;
1920 
1921  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1922  CopyValueToVirtualRegister(V, Reg);
1923 }
1924 
1926  const BasicBlock *FromBB) {
1927  // The operands of the setcc have to be in this block. We don't know
1928  // how to export them from some other block.
1929  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1930  // Can export from current BB.
1931  if (VI->getParent() == FromBB)
1932  return true;
1933 
1934  // Is already exported, noop.
1935  return FuncInfo.isExportedInst(V);
1936  }
1937 
1938  // If this is an argument, we can export it if the BB is the entry block or
1939  // if it is already exported.
1940  if (isa<Argument>(V)) {
1941  if (FromBB == &FromBB->getParent()->getEntryBlock())
1942  return true;
1943 
1944  // Otherwise, can only export this if it is already exported.
1945  return FuncInfo.isExportedInst(V);
1946  }
1947 
1948  // Otherwise, constants can always be exported.
1949  return true;
1950 }
1951 
1952 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1954 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1955  const MachineBasicBlock *Dst) const {
1956  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1957  const BasicBlock *SrcBB = Src->getBasicBlock();
1958  const BasicBlock *DstBB = Dst->getBasicBlock();
1959  if (!BPI) {
1960  // If BPI is not available, set the default probability as 1 / N, where N is
1961  // the number of successors.
1962  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1963  return BranchProbability(1, SuccSize);
1964  }
1965  return BPI->getEdgeProbability(SrcBB, DstBB);
1966 }
1967 
1968 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1969  MachineBasicBlock *Dst,
1970  BranchProbability Prob) {
1971  if (!FuncInfo.BPI)
1972  Src->addSuccessorWithoutProb(Dst);
1973  else {
1974  if (Prob.isUnknown())
1975  Prob = getEdgeProbability(Src, Dst);
1976  Src->addSuccessor(Dst, Prob);
1977  }
1978 }
1979 
1980 static bool InBlock(const Value *V, const BasicBlock *BB) {
1981  if (const Instruction *I = dyn_cast<Instruction>(V))
1982  return I->getParent() == BB;
1983  return true;
1984 }
1985 
1986 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1987 /// This function emits a branch and is used at the leaves of an OR or an
1988 /// AND operator tree.
1989 void
1991  MachineBasicBlock *TBB,
1992  MachineBasicBlock *FBB,
1993  MachineBasicBlock *CurBB,
1994  MachineBasicBlock *SwitchBB,
1995  BranchProbability TProb,
1996  BranchProbability FProb,
1997  bool InvertCond) {
1998  const BasicBlock *BB = CurBB->getBasicBlock();
1999 
2000  // If the leaf of the tree is a comparison, merge the condition into
2001  // the caseblock.
2002  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2003  // The operands of the cmp have to be in this block. We don't know
2004  // how to export them from some other block. If this is the first block
2005  // of the sequence, no exporting is needed.
2006  if (CurBB == SwitchBB ||
2007  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2008  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2009  ISD::CondCode Condition;
2010  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2011  ICmpInst::Predicate Pred =
2012  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2013  Condition = getICmpCondCode(Pred);
2014  } else {
2015  const FCmpInst *FC = cast<FCmpInst>(Cond);
2016  FCmpInst::Predicate Pred =
2017  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2018  Condition = getFCmpCondCode(Pred);
2019  if (TM.Options.NoNaNsFPMath)
2020  Condition = getFCmpCodeWithoutNaN(Condition);
2021  }
2022 
2023  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2024  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2025  SwitchCases.push_back(CB);
2026  return;
2027  }
2028  }
2029 
2030  // Create a CaseBlock record representing this branch.
2031  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2032  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2033  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2034  SwitchCases.push_back(CB);
2035 }
2036 
2038  MachineBasicBlock *TBB,
2039  MachineBasicBlock *FBB,
2040  MachineBasicBlock *CurBB,
2041  MachineBasicBlock *SwitchBB,
2043  BranchProbability TProb,
2044  BranchProbability FProb,
2045  bool InvertCond) {
2046  // Skip over not part of the tree and remember to invert op and operands at
2047  // next level.
2048  Value *NotCond;
2049  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2050  InBlock(NotCond, CurBB->getBasicBlock())) {
2051  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2052  !InvertCond);
2053  return;
2054  }
2055 
2056  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2057  // Compute the effective opcode for Cond, taking into account whether it needs
2058  // to be inverted, e.g.
2059  // and (not (or A, B)), C
2060  // gets lowered as
2061  // and (and (not A, not B), C)
2062  unsigned BOpc = 0;
2063  if (BOp) {
2064  BOpc = BOp->getOpcode();
2065  if (InvertCond) {
2066  if (BOpc == Instruction::And)
2067  BOpc = Instruction::Or;
2068  else if (BOpc == Instruction::Or)
2069  BOpc = Instruction::And;
2070  }
2071  }
2072 
2073  // If this node is not part of the or/and tree, emit it as a branch.
2074  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
2075  BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
2076  BOp->getParent() != CurBB->getBasicBlock() ||
2077  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
2078  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
2079  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2080  TProb, FProb, InvertCond);
2081  return;
2082  }
2083 
2084  // Create TmpBB after CurBB.
2085  MachineFunction::iterator BBI(CurBB);
2086  MachineFunction &MF = DAG.getMachineFunction();
2088  CurBB->getParent()->insert(++BBI, TmpBB);
2089 
2090  if (Opc == Instruction::Or) {
2091  // Codegen X | Y as:
2092  // BB1:
2093  // jmp_if_X TBB
2094  // jmp TmpBB
2095  // TmpBB:
2096  // jmp_if_Y TBB
2097  // jmp FBB
2098  //
2099 
2100  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2101  // The requirement is that
2102  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2103  // = TrueProb for original BB.
2104  // Assuming the original probabilities are A and B, one choice is to set
2105  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2106  // A/(1+B) and 2B/(1+B). This choice assumes that
2107  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2108  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2109  // TmpBB, but the math is more complicated.
2110 
2111  auto NewTrueProb = TProb / 2;
2112  auto NewFalseProb = TProb / 2 + FProb;
2113  // Emit the LHS condition.
2114  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
2115  NewTrueProb, NewFalseProb, InvertCond);
2116 
2117  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2118  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2119  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2120  // Emit the RHS condition into TmpBB.
2121  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2122  Probs[0], Probs[1], InvertCond);
2123  } else {
2124  assert(Opc == Instruction::And && "Unknown merge op!");
2125  // Codegen X & Y as:
2126  // BB1:
2127  // jmp_if_X TmpBB
2128  // jmp FBB
2129  // TmpBB:
2130  // jmp_if_Y TBB
2131  // jmp FBB
2132  //
2133  // This requires creation of TmpBB after CurBB.
2134 
2135  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2136  // The requirement is that
2137  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2138  // = FalseProb for original BB.
2139  // Assuming the original probabilities are A and B, one choice is to set
2140  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2141  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2142  // TrueProb for BB1 * FalseProb for TmpBB.
2143 
2144  auto NewTrueProb = TProb + FProb / 2;
2145  auto NewFalseProb = FProb / 2;
2146  // Emit the LHS condition.
2147  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
2148  NewTrueProb, NewFalseProb, InvertCond);
2149 
2150  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2151  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2152  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2153  // Emit the RHS condition into TmpBB.
2154  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2155  Probs[0], Probs[1], InvertCond);
2156  }
2157 }
2158 
2159 /// If the set of cases should be emitted as a series of branches, return true.
2160 /// If we should emit this as a bunch of and/or'd together conditions, return
2161 /// false.
2162 bool
2163 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2164  if (Cases.size() != 2) return true;
2165 
2166  // If this is two comparisons of the same values or'd or and'd together, they
2167  // will get folded into a single comparison, so don't emit two blocks.
2168  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2169  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2170  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2171  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2172  return false;
2173  }
2174 
2175  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2176  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2177  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2178  Cases[0].CC == Cases[1].CC &&
2179  isa<Constant>(Cases[0].CmpRHS) &&
2180  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2181  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2182  return false;
2183  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2184  return false;
2185  }
2186 
2187  return true;
2188 }
2189 
2190 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2191  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2192 
2193  // Update machine-CFG edges.
2194  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2195 
2196  if (I.isUnconditional()) {
2197  // Update machine-CFG edges.
2198  BrMBB->addSuccessor(Succ0MBB);
2199 
2200  // If this is not a fall-through branch or optimizations are switched off,
2201  // emit the branch.
2202  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2203  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2204  MVT::Other, getControlRoot(),
2205  DAG.getBasicBlock(Succ0MBB)));
2206 
2207  return;
2208  }
2209 
2210  // If this condition is one of the special cases we handle, do special stuff
2211  // now.
2212  const Value *CondVal = I.getCondition();
2213  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2214 
2215  // If this is a series of conditions that are or'd or and'd together, emit
2216  // this as a sequence of branches instead of setcc's with and/or operations.
2217  // As long as jumps are not expensive, this should improve performance.
2218  // For example, instead of something like:
2219  // cmp A, B
2220  // C = seteq
2221  // cmp D, E
2222  // F = setle
2223  // or C, F
2224  // jnz foo
2225  // Emit:
2226  // cmp A, B
2227  // je foo
2228  // cmp D, E
2229  // jle foo
2230  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2231  Instruction::BinaryOps Opcode = BOp->getOpcode();
2232  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2234  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2235  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2236  Opcode,
2237  getEdgeProbability(BrMBB, Succ0MBB),
2238  getEdgeProbability(BrMBB, Succ1MBB),
2239  /*InvertCond=*/false);
2240  // If the compares in later blocks need to use values not currently
2241  // exported from this block, export them now. This block should always
2242  // be the first entry.
2243  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2244 
2245  // Allow some cases to be rejected.
2246  if (ShouldEmitAsBranches(SwitchCases)) {
2247  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2248  ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2249  ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2250  }
2251 
2252  // Emit the branch for this block.
2253  visitSwitchCase(SwitchCases[0], BrMBB);
2254  SwitchCases.erase(SwitchCases.begin());
2255  return;
2256  }
2257 
2258  // Okay, we decided not to do this, remove any inserted MBB's and clear
2259  // SwitchCases.
2260  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2261  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2262 
2263  SwitchCases.clear();
2264  }
2265  }
2266 
2267  // Create a CaseBlock record representing this branch.
2268  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2269  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2270 
2271  // Use visitSwitchCase to actually insert the fast branch sequence for this
2272  // cond branch.
2273  visitSwitchCase(CB, BrMBB);
2274 }
2275 
2276 /// visitSwitchCase - Emits the necessary code to represent a single node in
2277 /// the binary search tree resulting from lowering a switch instruction.
2279  MachineBasicBlock *SwitchBB) {
2280  SDValue Cond;
2281  SDValue CondLHS = getValue(CB.CmpLHS);
2282  SDLoc dl = CB.DL;
2283 
2284  // Build the setcc now.
2285  if (!CB.CmpMHS) {
2286  // Fold "(X == true)" to X and "(X == false)" to !X to
2287  // handle common cases produced by branch lowering.
2288  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2289  CB.CC == ISD::SETEQ)
2290  Cond = CondLHS;
2291  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2292  CB.CC == ISD::SETEQ) {
2293  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2294  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2295  } else
2296  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2297  } else {
2298  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2299 
2300  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2301  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2302 
2303  SDValue CmpOp = getValue(CB.CmpMHS);
2304  EVT VT = CmpOp.getValueType();
2305 
2306  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2307  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2308  ISD::SETLE);
2309  } else {
2310  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2311  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2312  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2313  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2314  }
2315  }
2316 
2317  // Update successor info
2318  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2319  // TrueBB and FalseBB are always different unless the incoming IR is
2320  // degenerate. This only happens when running llc on weird IR.
2321  if (CB.TrueBB != CB.FalseBB)
2322  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2323  SwitchBB->normalizeSuccProbs();
2324 
2325  // If the lhs block is the next block, invert the condition so that we can
2326  // fall through to the lhs instead of the rhs block.
2327  if (CB.TrueBB == NextBlock(SwitchBB)) {
2328  std::swap(CB.TrueBB, CB.FalseBB);
2329  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2330  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2331  }
2332 
2333  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2334  MVT::Other, getControlRoot(), Cond,
2335  DAG.getBasicBlock(CB.TrueBB));
2336 
2337  // Insert the false branch. Do this even if it's a fall through branch,
2338  // this makes it easier to do DAG optimizations which require inverting
2339  // the branch condition.
2340  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2341  DAG.getBasicBlock(CB.FalseBB));
2342 
2343  DAG.setRoot(BrCond);
2344 }
2345 
2346 /// visitJumpTable - Emit JumpTable node in the current MBB
2348  // Emit the code for the jump table
2349  assert(JT.Reg != -1U && "Should lower JT Header first!");
2351  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2352  JT.Reg, PTy);
2353  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2354  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2355  MVT::Other, Index.getValue(1),
2356  Table, Index);
2357  DAG.setRoot(BrJumpTable);
2358 }
2359 
2360 /// visitJumpTableHeader - This function emits necessary code to produce index
2361 /// in the JumpTable from switch case.
2363  JumpTableHeader &JTH,
2364  MachineBasicBlock *SwitchBB) {
2365  SDLoc dl = getCurSDLoc();
2366 
2367  // Subtract the lowest switch case value from the value being switched on and
2368  // conditional branch to default mbb if the result is greater than the
2369  // difference between smallest and largest cases.
2370  SDValue SwitchOp = getValue(JTH.SValue);
2371  EVT VT = SwitchOp.getValueType();
2372  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2373  DAG.getConstant(JTH.First, dl, VT));
2374 
2375  // The SDNode we just created, which holds the value being switched on minus
2376  // the smallest case value, needs to be copied to a virtual register so it
2377  // can be used as an index into the jump table in a subsequent basic block.
2378  // This value may be smaller or larger than the target's pointer type, and
2379  // therefore require extension or truncating.
2380  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2381  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2382 
2383  unsigned JumpTableReg =
2384  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2385  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2386  JumpTableReg, SwitchOp);
2387  JT.Reg = JumpTableReg;
2388 
2389  // Emit the range check for the jump table, and branch to the default block
2390  // for the switch statement if the value being switched on exceeds the largest
2391  // case in the switch.
2392  SDValue CMP = DAG.getSetCC(
2393  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2394  Sub.getValueType()),
2395  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2396 
2397  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2398  MVT::Other, CopyTo, CMP,
2399  DAG.getBasicBlock(JT.Default));
2400 
2401  // Avoid emitting unnecessary branches to the next block.
2402  if (JT.MBB != NextBlock(SwitchBB))
2403  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2404  DAG.getBasicBlock(JT.MBB));
2405 
2406  DAG.setRoot(BrCond);
2407 }
2408 
2409 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2410 /// variable if there exists one.
2412  SDValue &Chain) {
2413  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2414  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2415  MachineFunction &MF = DAG.getMachineFunction();
2416  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2417  MachineSDNode *Node =
2418  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2419  if (Global) {
2420  MachinePointerInfo MPInfo(Global);
2424  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2425  DAG.setNodeMemRefs(Node, {MemRef});
2426  }
2427  return SDValue(Node, 0);
2428 }
2429 
2430 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2431 /// tail spliced into a stack protector check success bb.
2432 ///
2433 /// For a high level explanation of how this fits into the stack protector
2434 /// generation see the comment on the declaration of class
2435 /// StackProtectorDescriptor.
2436 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2437  MachineBasicBlock *ParentBB) {
2438 
2439  // First create the loads to the guard/stack slot for the comparison.
2440  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2441  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2442 
2443  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2444  int FI = MFI.getStackProtectorIndex();
2445 
2446  SDValue Guard;
2447  SDLoc dl = getCurSDLoc();
2448  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2449  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2450  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2451 
2452  // Generate code to load the content of the guard slot.
2453  SDValue GuardVal = DAG.getLoad(
2454  PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2457 
2458  if (TLI.useStackGuardXorFP())
2459  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2460 
2461  // Retrieve guard check function, nullptr if instrumentation is inlined.
2462  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2463  // The target provides a guard check function to validate the guard value.
2464  // Generate a call to that function with the content of the guard slot as
2465  // argument.
2466  FunctionType *FnTy = GuardCheckFn->getFunctionType();
2467  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2468 
2471  Entry.Node = GuardVal;
2472  Entry.Ty = FnTy->getParamType(0);
2473  if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
2474  Entry.IsInReg = true;
2475  Args.push_back(Entry);
2476 
2478  CLI.setDebugLoc(getCurSDLoc())
2479  .setChain(DAG.getEntryNode())
2480  .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2481  getValue(GuardCheckFn), std::move(Args));
2482 
2483  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2484  DAG.setRoot(Result.second);
2485  return;
2486  }
2487 
2488  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2489  // Otherwise, emit a volatile load to retrieve the stack guard value.
2490  SDValue Chain = DAG.getEntryNode();
2491  if (TLI.useLoadStackGuardNode()) {
2492  Guard = getLoadStackGuard(DAG, dl, Chain);
2493  } else {
2494  const Value *IRGuard = TLI.getSDagStackGuard(M);
2495  SDValue GuardPtr = getValue(IRGuard);
2496 
2497  Guard =
2498  DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2500  }
2501 
2502  // Perform the comparison via a subtract/getsetcc.
2503  EVT VT = Guard.getValueType();
2504  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2505 
2506  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2507  *DAG.getContext(),
2508  Sub.getValueType()),
2509  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2510 
2511  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2512  // branch to failure MBB.
2513  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2514  MVT::Other, GuardVal.getOperand(0),
2515  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2516  // Otherwise branch to success MBB.
2517  SDValue Br = DAG.getNode(ISD::BR, dl,
2518  MVT::Other, BrCond,
2519  DAG.getBasicBlock(SPD.getSuccessMBB()));
2520 
2521  DAG.setRoot(Br);
2522 }
2523 
2524 /// Codegen the failure basic block for a stack protector check.
2525 ///
2526 /// A failure stack protector machine basic block consists simply of a call to
2527 /// __stack_chk_fail().
2528 ///
2529 /// For a high level explanation of how this fits into the stack protector
2530 /// generation see the comment on the declaration of class
2531 /// StackProtectorDescriptor.
2532 void
2533 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2534  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2535  SDValue Chain =
2536  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2537  None, false, getCurSDLoc(), false, false).second;
2538  DAG.setRoot(Chain);
2539 }
2540 
2541 /// visitBitTestHeader - This function emits necessary code to produce value
2542 /// suitable for "bit tests"
2544  MachineBasicBlock *SwitchBB) {
2545  SDLoc dl = getCurSDLoc();
2546 
2547  // Subtract the minimum value
2548  SDValue SwitchOp = getValue(B.SValue);
2549  EVT VT = SwitchOp.getValueType();
2550  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2551  DAG.getConstant(B.First, dl, VT));
2552 
2553  // Check range
2554  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2555  SDValue RangeCmp = DAG.getSetCC(
2556  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2557  Sub.getValueType()),
2558  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2559 
2560  // Determine the type of the test operands.
2561  bool UsePtrType = false;
2562  if (!TLI.isTypeLegal(VT))
2563  UsePtrType = true;
2564  else {
2565  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2566  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2567  // Switch table case range are encoded into series of masks.
2568  // Just use pointer type, it's guaranteed to fit.
2569  UsePtrType = true;
2570  break;
2571  }
2572  }
2573  if (UsePtrType) {
2574  VT = TLI.getPointerTy(DAG.getDataLayout());
2575  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2576  }
2577 
2578  B.RegVT = VT.getSimpleVT();
2579  B.Reg = FuncInfo.CreateReg(B.RegVT);
2580  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2581 
2582  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2583 
2584  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2585  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2586  SwitchBB->normalizeSuccProbs();
2587 
2588  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2589  MVT::Other, CopyTo, RangeCmp,
2590  DAG.getBasicBlock(B.Default));
2591 
2592  // Avoid emitting unnecessary branches to the next block.
2593  if (MBB != NextBlock(SwitchBB))
2594  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2595  DAG.getBasicBlock(MBB));
2596 
2597  DAG.setRoot(BrRange);
2598 }
2599 
2600 /// visitBitTestCase - this function produces one "bit test"
2602  MachineBasicBlock* NextMBB,
2603  BranchProbability BranchProbToNext,
2604  unsigned Reg,
2605  BitTestCase &B,
2606  MachineBasicBlock *SwitchBB) {
2607  SDLoc dl = getCurSDLoc();
2608  MVT VT = BB.RegVT;
2609  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2610  SDValue Cmp;
2611  unsigned PopCount = countPopulation(B.Mask);
2612  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2613  if (PopCount == 1) {
2614  // Testing for a single bit; just compare the shift count with what it
2615  // would need to be to shift a 1 bit in that position.
2616  Cmp = DAG.getSetCC(
2617  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2618  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2619  ISD::SETEQ);
2620  } else if (PopCount == BB.Range) {
2621  // There is only one zero bit in the range, test for it directly.
2622  Cmp = DAG.getSetCC(
2623  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2624  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2625  ISD::SETNE);
2626  } else {
2627  // Make desired shift
2628  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2629  DAG.getConstant(1, dl, VT), ShiftOp);
2630 
2631  // Emit bit tests and jumps
2632  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2633  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2634  Cmp = DAG.getSetCC(
2635  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2636  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2637  }
2638 
2639  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2640  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2641  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2642  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2643  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2644  // one as they are relative probabilities (and thus work more like weights),
2645  // and hence we need to normalize them to let the sum of them become one.
2646  SwitchBB->normalizeSuccProbs();
2647 
2648  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2649  MVT::Other, getControlRoot(),
2650  Cmp, DAG.getBasicBlock(B.TargetBB));
2651 
2652  // Avoid emitting unnecessary branches to the next block.
2653  if (NextMBB != NextBlock(SwitchBB))
2654  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2655  DAG.getBasicBlock(NextMBB));
2656 
2657  DAG.setRoot(BrAnd);
2658 }
2659 
2660 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2661  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2662 
2663  // Retrieve successors. Look through artificial IR level blocks like
2664  // catchswitch for successors.
2665  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2666  const BasicBlock *EHPadBB = I.getSuccessor(1);
2667 
2668  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2669  // have to do anything here to lower funclet bundles.
2671  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2672  "Cannot lower invokes with arbitrary operand bundles yet!");
2673 
2674  const Value *Callee(I.getCalledValue());
2675  const Function *Fn = dyn_cast<Function>(Callee);
2676  if (isa<InlineAsm>(Callee))
2677  visitInlineAsm(&I);
2678  else if (Fn && Fn->isIntrinsic()) {
2679  switch (Fn->getIntrinsicID()) {
2680  default:
2681  llvm_unreachable("Cannot invoke this intrinsic");
2682  case Intrinsic::donothing:
2683  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2684  break;
2685  case Intrinsic::experimental_patchpoint_void:
2686  case Intrinsic::experimental_patchpoint_i64:
2687  visitPatchpoint(&I, EHPadBB);
2688  break;
2689  case Intrinsic::experimental_gc_statepoint:
2690  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2691  break;
2692  }
2694  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2695  // Eventually we will support lowering the @llvm.experimental.deoptimize
2696  // intrinsic, and right now there are no plans to support other intrinsics
2697  // with deopt state.
2698  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2699  } else {
2700  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2701  }
2702 
2703  // If the value of the invoke is used outside of its defining block, make it
2704  // available as a virtual register.
2705  // We already took care of the exported value for the statepoint instruction
2706  // during call to the LowerStatepoint.
2707  if (!isStatepoint(I)) {
2708  CopyToExportRegsIfNeeded(&I);
2709  }
2710 
2712  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2713  BranchProbability EHPadBBProb =
2714  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2716  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2717 
2718  // Update successor info.
2719  addSuccessorWithProb(InvokeMBB, Return);
2720  for (auto &UnwindDest : UnwindDests) {
2721  UnwindDest.first->setIsEHPad();
2722  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2723  }
2724  InvokeMBB->normalizeSuccProbs();
2725 
2726  // Drop into normal successor.
2727  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2728  DAG.getBasicBlock(Return)));
2729 }
2730 
2731 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2732  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2733 
2734  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2735  // have to do anything here to lower funclet bundles.
2737  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2738  "Cannot lower callbrs with arbitrary operand bundles yet!");
2739 
2740  assert(isa<InlineAsm>(I.getCalledValue()) &&
2741  "Only know how to handle inlineasm callbr");
2742  visitInlineAsm(&I);
2743 
2744  // Retrieve successors.
2745  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2746 
2747  // Update successor info.
2748  addSuccessorWithProb(CallBrMBB, Return);
2749  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2750  MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2751  addSuccessorWithProb(CallBrMBB, Target);
2752  }
2753  CallBrMBB->normalizeSuccProbs();
2754 
2755  // Drop into default successor.
2756  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2757  MVT::Other, getControlRoot(),
2758  DAG.getBasicBlock(Return)));
2759 }
2760 
2761 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2762  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2763 }
2764 
2765 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2766  assert(FuncInfo.MBB->isEHPad() &&
2767  "Call to landingpad not in landing pad!");
2768 
2769  // If there aren't registers to copy the values into (e.g., during SjLj
2770  // exceptions), then don't bother to create these DAG nodes.
2771  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2772  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2773  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2774  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2775  return;
2776 
2777  // If landingpad's return type is token type, we don't create DAG nodes
2778  // for its exception pointer and selector value. The extraction of exception
2779  // pointer or selector value from token type landingpads is not currently
2780  // supported.
2781  if (LP.getType()->isTokenTy())
2782  return;
2783 
2785  SDLoc dl = getCurSDLoc();
2786  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2787  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2788 
2789  // Get the two live-in registers as SDValues. The physregs have already been
2790  // copied into virtual registers.
2791  SDValue Ops[2];
2792  if (FuncInfo.ExceptionPointerVirtReg) {
2793  Ops[0] = DAG.getZExtOrTrunc(
2794  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2795  FuncInfo.ExceptionPointerVirtReg,
2796  TLI.getPointerTy(DAG.getDataLayout())),
2797  dl, ValueVTs[0]);
2798  } else {
2799  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2800  }
2801  Ops[1] = DAG.getZExtOrTrunc(
2802  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2803  FuncInfo.ExceptionSelectorVirtReg,
2804  TLI.getPointerTy(DAG.getDataLayout())),
2805  dl, ValueVTs[1]);
2806 
2807  // Merge into one.
2808  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2809  DAG.getVTList(ValueVTs), Ops);
2810  setValue(&LP, Res);
2811 }
2812 
2813 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2814 #ifndef NDEBUG
2815  for (const CaseCluster &CC : Clusters)
2816  assert(CC.Low == CC.High && "Input clusters must be single-case");
2817 #endif
2818 
2819  llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) {
2820  return a.Low->getValue().slt(b.Low->getValue());
2821  });
2822 
2823  // Merge adjacent clusters with the same destination.
2824  const unsigned N = Clusters.size();
2825  unsigned DstIndex = 0;
2826  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2827  CaseCluster &CC = Clusters[SrcIndex];
2828  const ConstantInt *CaseVal = CC.Low;
2829  MachineBasicBlock *Succ = CC.MBB;
2830 
2831  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2832  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2833  // If this case has the same successor and is a neighbour, merge it into
2834  // the previous cluster.
2835  Clusters[DstIndex - 1].High = CaseVal;
2836  Clusters[DstIndex - 1].Prob += CC.Prob;
2837  } else {
2838  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2839  sizeof(Clusters[SrcIndex]));
2840  }
2841  }
2842  Clusters.resize(DstIndex);
2843 }
2844 
2846  MachineBasicBlock *Last) {
2847  // Update JTCases.
2848  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2849  if (JTCases[i].first.HeaderBB == First)
2850  JTCases[i].first.HeaderBB = Last;
2851 
2852  // Update BitTestCases.
2853  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2854  if (BitTestCases[i].Parent == First)
2855  BitTestCases[i].Parent = Last;
2856 }
2857 
2858 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2859  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2860 
2861  // Update machine-CFG edges with unique successors.
2863  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2864  BasicBlock *BB = I.getSuccessor(i);
2865  bool Inserted = Done.insert(BB).second;
2866  if (!Inserted)
2867  continue;
2868 
2869  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2870  addSuccessorWithProb(IndirectBrMBB, Succ);
2871  }
2872  IndirectBrMBB->normalizeSuccProbs();
2873 
2874  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2875  MVT::Other, getControlRoot(),
2876  getValue(I.getAddress())));
2877 }
2878 
2879 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2880  if (!DAG.getTarget().Options.TrapUnreachable)
2881  return;
2882 
2883  // We may be able to ignore unreachable behind a noreturn call.
2885  const BasicBlock &BB = *I.getParent();
2886  if (&I != &BB.front()) {
2888  std::prev(BasicBlock::const_iterator(&I));
2889  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2890  if (Call->doesNotReturn())
2891  return;
2892  }
2893  }
2894  }
2895 
2896  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2897 }
2898 
2899 void SelectionDAGBuilder::visitFSub(const User &I) {
2900  // -0.0 - X --> fneg
2901  Type *Ty = I.getType();
2902  if (isa<Constant>(I.getOperand(0)) &&
2904  SDValue Op2 = getValue(I.getOperand(1));
2905  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2906  Op2.getValueType(), Op2));
2907  return;
2908  }
2909 
2910  visitBinary(I, ISD::FSUB);
2911 }
2912 
2913 /// Checks if the given instruction performs a vector reduction, in which case
2914 /// we have the freedom to alter the elements in the result as long as the
2915 /// reduction of them stays unchanged.
2916 static bool isVectorReductionOp(const User *I) {
2917  const Instruction *Inst = dyn_cast<Instruction>(I);
2918  if (!Inst || !Inst->getType()->isVectorTy())
2919  return false;
2920 
2921  auto OpCode = Inst->getOpcode();
2922  switch (OpCode) {
2923  case Instruction::Add:
2924  case Instruction::Mul:
2925  case Instruction::And:
2926  case Instruction::Or:
2927  case Instruction::Xor:
2928  break;
2929  case Instruction::FAdd:
2930  case Instruction::FMul:
2931  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2932  if (FPOp->getFastMathFlags().isFast())
2933  break;
2935  default:
2936  return false;
2937  }
2938 
2939  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2940  // Ensure the reduction size is a power of 2.
2941  if (!isPowerOf2_32(ElemNum))
2942  return false;
2943 
2944  unsigned ElemNumToReduce = ElemNum;
2945 
2946  // Do DFS search on the def-use chain from the given instruction. We only
2947  // allow four kinds of operations during the search until we reach the
2948  // instruction that extracts the first element from the vector:
2949  //
2950  // 1. The reduction operation of the same opcode as the given instruction.
2951  //
2952  // 2. PHI node.
2953  //
2954  // 3. ShuffleVector instruction together with a reduction operation that
2955  // does a partial reduction.
2956  //
2957  // 4. ExtractElement that extracts the first element from the vector, and we
2958  // stop searching the def-use chain here.
2959  //
2960  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2961  // from 1-3 to the stack to continue the DFS. The given instruction is not
2962  // a reduction operation if we meet any other instructions other than those
2963  // listed above.
2964 
2965  SmallVector<const User *, 16> UsersToVisit{Inst};
2967  bool ReduxExtracted = false;
2968 
2969  while (!UsersToVisit.empty()) {
2970  auto User = UsersToVisit.back();
2971  UsersToVisit.pop_back();
2972  if (!Visited.insert(User).second)
2973  continue;
2974 
2975  for (const auto &U : User->users()) {
2976  auto Inst = dyn_cast<Instruction>(U);
2977  if (!Inst)
2978  return false;
2979 
2980  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2981  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2982  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2983  return false;
2984  UsersToVisit.push_back(U);
2985  } else if (const ShuffleVectorInst *ShufInst =
2986  dyn_cast<ShuffleVectorInst>(U)) {
2987  // Detect the following pattern: A ShuffleVector instruction together
2988  // with a reduction that do partial reduction on the first and second
2989  // ElemNumToReduce / 2 elements, and store the result in
2990  // ElemNumToReduce / 2 elements in another vector.
2991 
2992  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2993  if (ResultElements < ElemNum)
2994  return false;
2995 
2996  if (ElemNumToReduce == 1)
2997  return false;
2998  if (!isa<UndefValue>(U->getOperand(1)))
2999  return false;
3000  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
3001  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
3002  return false;
3003  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
3004  if (ShufInst->getMaskValue(i) != -1)
3005  return false;
3006 
3007  // There is only one user of this ShuffleVector instruction, which
3008  // must be a reduction operation.
3009  if (!U->hasOneUse())
3010  return false;
3011 
3012  auto U2 = dyn_cast<Instruction>(*U->user_begin());
3013  if (!U2 || U2->getOpcode() != OpCode)
3014  return false;
3015 
3016  // Check operands of the reduction operation.
3017  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
3018  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
3019  UsersToVisit.push_back(U2);
3020  ElemNumToReduce /= 2;
3021  } else
3022  return false;
3023  } else if (isa<ExtractElementInst>(U)) {
3024  // At this moment we should have reduced all elements in the vector.
3025  if (ElemNumToReduce != 1)
3026  return false;
3027 
3028  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
3029  if (!Val || !Val->isZero())
3030  return false;
3031 
3032  ReduxExtracted = true;
3033  } else
3034  return false;
3035  }
3036  }
3037  return ReduxExtracted;
3038 }
3039 
3040 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3041  SDNodeFlags Flags;
3042 
3043  SDValue Op = getValue(I.getOperand(0));
3044  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3045  Op, Flags);
3046  setValue(&I, UnNodeValue);
3047 }
3048 
3049 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3050  SDNodeFlags Flags;
3051  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3052  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3053  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3054  }
3055  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
3056  Flags.setExact(ExactOp->isExact());
3057  }
3058  if (isVectorReductionOp(&I)) {
3059  Flags.setVectorReduction(true);
3060  LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
3061  }
3062 
3063  SDValue Op1 = getValue(I.getOperand(0));
3064  SDValue Op2 = getValue(I.getOperand(1));
3065  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3066  Op1, Op2, Flags);
3067  setValue(&I, BinNodeValue);
3068 }
3069 
3070 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3071  SDValue Op1 = getValue(I.getOperand(0));
3072  SDValue Op2 = getValue(I.getOperand(1));
3073 
3074  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3075  Op1.getValueType(), DAG.getDataLayout());
3076 
3077  // Coerce the shift amount to the right type if we can.
3078  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3079  unsigned ShiftSize = ShiftTy.getSizeInBits();
3080  unsigned Op2Size = Op2.getValueSizeInBits();
3081  SDLoc DL = getCurSDLoc();
3082 
3083  // If the operand is smaller than the shift count type, promote it.
3084  if (ShiftSize > Op2Size)
3085  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3086 
3087  // If the operand is larger than the shift count type but the shift
3088  // count type has enough bits to represent any shift value, truncate
3089  // it now. This is a common case and it exposes the truncate to
3090  // optimization early.
3091  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3092  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3093  // Otherwise we'll need to temporarily settle for some other convenient
3094  // type. Type legalization will make adjustments once the shiftee is split.
3095  else
3096  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3097  }
3098 
3099  bool nuw = false;
3100  bool nsw = false;
3101  bool exact = false;
3102 
3103  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3104 
3105  if (const OverflowingBinaryOperator *OFBinOp =
3106  dyn_cast<const OverflowingBinaryOperator>(&I)) {
3107  nuw = OFBinOp->hasNoUnsignedWrap();
3108  nsw = OFBinOp->hasNoSignedWrap();
3109  }
3110  if (const PossiblyExactOperator *ExactOp =
3111  dyn_cast<const PossiblyExactOperator>(&I))
3112  exact = ExactOp->isExact();
3113  }
3114  SDNodeFlags Flags;
3115  Flags.setExact(exact);
3116  Flags.setNoSignedWrap(nsw);
3117  Flags.setNoUnsignedWrap(nuw);
3118  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3119  Flags);
3120  setValue(&I, Res);
3121 }
3122 
3123 void SelectionDAGBuilder::visitSDiv(const User &I) {
3124  SDValue Op1 = getValue(I.getOperand(0));
3125  SDValue Op2 = getValue(I.getOperand(1));
3126 
3127  SDNodeFlags Flags;
3128  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3129  cast<PossiblyExactOperator>(&I)->isExact());
3130  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3131  Op2, Flags));
3132 }
3133 
3134 void SelectionDAGBuilder::visitICmp(const User &I) {
3136  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3137  predicate = IC->getPredicate();
3138  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3139  predicate = ICmpInst::Predicate(IC->getPredicate());
3140  SDValue Op1 = getValue(I.getOperand(0));
3141  SDValue Op2 = getValue(I.getOperand(1));
3142  ISD::CondCode Opcode = getICmpCondCode(predicate);
3143 
3144  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3145  I.getType());
3146  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3147 }
3148 
3149 void SelectionDAGBuilder::visitFCmp(const User &I) {
3151  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3152  predicate = FC->getPredicate();
3153  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3154  predicate = FCmpInst::Predicate(FC->getPredicate());
3155  SDValue Op1 = getValue(I.getOperand(0));
3156  SDValue Op2 = getValue(I.getOperand(1));
3157 
3158  ISD::CondCode Condition = getFCmpCondCode(predicate);
3159  auto *FPMO = dyn_cast<FPMathOperator>(&I);
3160  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
3161  Condition = getFCmpCodeWithoutNaN(Condition);
3162 
3163  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3164  I.getType());
3165  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3166 }
3167 
3168 // Check if the condition of the select has one use or two users that are both
3169 // selects with the same condition.
3170 static bool hasOnlySelectUsers(const Value *Cond) {
3171  return llvm::all_of(Cond->users(), [](const Value *V) {
3172  return isa<SelectInst>(V);
3173  });
3174 }
3175 
3176 void SelectionDAGBuilder::visitSelect(const User &I) {
3179  ValueVTs);
3180  unsigned NumValues = ValueVTs.size();
3181  if (NumValues == 0) return;
3182 
3183  SmallVector<SDValue, 4> Values(NumValues);
3184  SDValue Cond = getValue(I.getOperand(0));
3185  SDValue LHSVal = getValue(I.getOperand(1));
3186  SDValue RHSVal = getValue(I.getOperand(2));
3187  auto BaseOps = {Cond};
3188  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
3190 
3191  // Min/max matching is only viable if all output VTs are the same.
3192  if (is_splat(ValueVTs)) {
3193  EVT VT = ValueVTs[0];
3194  LLVMContext &Ctx = *DAG.getContext();
3195  auto &TLI = DAG.getTargetLoweringInfo();
3196 
3197  // We care about the legality of the operation after it has been type
3198  // legalized.
3199  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
3200  VT != TLI.getTypeToTransformTo(Ctx, VT))
3201  VT = TLI.getTypeToTransformTo(Ctx, VT);
3202 
3203  // If the vselect is legal, assume we want to leave this as a vector setcc +
3204  // vselect. Otherwise, if this is going to be scalarized, we want to see if
3205  // min/max is legal on the scalar type.
3206  bool UseScalarMinMax = VT.isVector() &&
3207  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3208 
3209  Value *LHS, *RHS;
3210  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3212  switch (SPR.Flavor) {
3213  case SPF_UMAX: Opc = ISD::UMAX; break;
3214  case SPF_UMIN: Opc = ISD::UMIN; break;
3215  case SPF_SMAX: Opc = ISD::SMAX; break;
3216  case SPF_SMIN: Opc = ISD::SMIN; break;
3217  case SPF_FMINNUM:
3218  switch (SPR.NaNBehavior) {
3219  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3220  case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3221  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3222  case SPNB_RETURNS_ANY: {
3223  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3224  Opc = ISD::FMINNUM;
3225  else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3226  Opc = ISD::FMINIMUM;
3227  else if (UseScalarMinMax)
3228  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3230  break;
3231  }
3232  }
3233  break;
3234  case SPF_FMAXNUM:
3235  switch (SPR.NaNBehavior) {
3236  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3237  case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3238  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3239  case SPNB_RETURNS_ANY:
3240 
3241  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3242  Opc = ISD::FMAXNUM;
3243  else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3244  Opc = ISD::FMAXIMUM;
3245  else if (UseScalarMinMax)
3246  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3248  break;
3249  }
3250  break;
3251  default: break;
3252  }
3253 
3254  if (Opc != ISD::DELETED_NODE &&
3255  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3256  (UseScalarMinMax &&
3257  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3258  // If the underlying comparison instruction is used by any other
3259  // instruction, the consumed instructions won't be destroyed, so it is
3260  // not profitable to convert to a min/max.
3261  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3262  OpCode = Opc;
3263  LHSVal = getValue(LHS);
3264  RHSVal = getValue(RHS);
3265  BaseOps = {};
3266  }
3267  }
3268 
3269  for (unsigned i = 0; i != NumValues; ++i) {
3270  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3271  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3272  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3273  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3274  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3275  Ops);
3276  }
3277 
3278  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3279  DAG.getVTList(ValueVTs), Values));
3280 }
3281 
3282 void SelectionDAGBuilder::visitTrunc(const User &I) {
3283  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3284  SDValue N = getValue(I.getOperand(0));
3285  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3286  I.getType());
3287  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3288 }
3289 
3290 void SelectionDAGBuilder::visitZExt(const User &I) {
3291  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3292  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3293  SDValue N = getValue(I.getOperand(0));
3294  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3295  I.getType());
3296  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3297 }
3298 
3299 void SelectionDAGBuilder::visitSExt(const User &I) {
3300  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3301  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3302  SDValue N = getValue(I.getOperand(0));
3303  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3304  I.getType());
3305  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3306 }
3307 
3308 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3309  // FPTrunc is never a no-op cast, no need to check
3310  SDValue N = getValue(I.getOperand(0));
3311  SDLoc dl = getCurSDLoc();
3312  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3313  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3314  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3315  DAG.getTargetConstant(
3316  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3317 }
3318 
3319 void SelectionDAGBuilder::visitFPExt(const User &I) {
3320  // FPExt is never a no-op cast, no need to check
3321  SDValue N = getValue(I.getOperand(0));
3322  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3323  I.getType());
3324  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3325 }
3326 
3327 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3328  // FPToUI is never a no-op cast, no need to check
3329  SDValue N = getValue(I.getOperand(0));
3330  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3331  I.getType());
3332  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3333 }
3334 
3335 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3336  // FPToSI is never a no-op cast, no need to check
3337  SDValue N = getValue(I.getOperand(0));
3338  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3339  I.getType());
3340  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3341 }
3342 
3343 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3344  // UIToFP is never a no-op cast, no need to check
3345  SDValue N = getValue(I.getOperand(0));
3346  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3347  I.getType());
3348  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3349 }
3350 
3351 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3352  // SIToFP is never a no-op cast, no need to check
3353  SDValue N = getValue(I.getOperand(0));
3354  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3355  I.getType());
3356  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3357 }
3358 
3359 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3360  // What to do depends on the size of the integer and the size of the pointer.
3361  // We can either truncate, zero extend, or no-op, accordingly.
3362  SDValue N = getValue(I.getOperand(0));
3363  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3364  I.getType());
3365  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3366 }
3367 
3368 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3369  // What to do depends on the size of the integer and the size of the pointer.
3370  // We can either truncate, zero extend, or no-op, accordingly.
3371  SDValue N = getValue(I.getOperand(0));
3372  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3373  I.getType());
3374  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3375 }
3376 
3377 void SelectionDAGBuilder::visitBitCast(const User &I) {
3378  SDValue N = getValue(I.getOperand(0));
3379  SDLoc dl = getCurSDLoc();
3380  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3381  I.getType());
3382 
3383  // BitCast assures us that source and destination are the same size so this is
3384  // either a BITCAST or a no-op.
3385  if (DestVT != N.getValueType())
3386  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3387  DestVT, N)); // convert types.
3388  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3389  // might fold any kind of constant expression to an integer constant and that
3390  // is not what we are looking for. Only recognize a bitcast of a genuine
3391  // constant integer as an opaque constant.
3392  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3393  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3394  /*isOpaque*/true));
3395  else
3396  setValue(&I, N); // noop cast.
3397 }
3398 
3399 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3400  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3401  const Value *SV = I.getOperand(0);
3402  SDValue N = getValue(SV);
3403  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3404 
3405  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3406  unsigned DestAS = I.getType()->getPointerAddressSpace();
3407 
3408  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3409  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3410 
3411  setValue(&I, N);
3412 }
3413 
3414 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3415  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3416  SDValue InVec = getValue(I.getOperand(0));
3417  SDValue InVal = getValue(I.getOperand(1));
3418  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3419  TLI.getVectorIdxTy(DAG.getDataLayout()));
3420  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3421  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3422  InVec, InVal, InIdx));
3423 }
3424 
3425 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3426  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3427  SDValue InVec = getValue(I.getOperand(0));
3428  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3429  TLI.getVectorIdxTy(DAG.getDataLayout()));
3430  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3431  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3432  InVec, InIdx));
3433 }
3434 
3435 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3436  SDValue Src1 = getValue(I.getOperand(0));
3437  SDValue Src2 = getValue(I.getOperand(1));
3438  SDLoc DL = getCurSDLoc();
3439 
3441  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3442  unsigned MaskNumElts = Mask.size();
3443 
3444  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3445  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3446  EVT SrcVT = Src1.getValueType();
3447  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3448 
3449  if (SrcNumElts == MaskNumElts) {
3450  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3451  return;
3452  }
3453 
3454  // Normalize the shuffle vector since mask and vector length don't match.
3455  if (SrcNumElts < MaskNumElts) {
3456  // Mask is longer than the source vectors. We can use concatenate vector to
3457  // make the mask and vectors lengths match.
3458 
3459  if (MaskNumElts % SrcNumElts == 0) {
3460  // Mask length is a multiple of the source vector length.
3461  // Check if the shuffle is some kind of concatenation of the input
3462  // vectors.
3463  unsigned NumConcat = MaskNumElts / SrcNumElts;
3464  bool IsConcat = true;
3465  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3466  for (unsigned i = 0; i != MaskNumElts; ++i) {
3467  int Idx = Mask[i];
3468  if (Idx < 0)
3469  continue;
3470  // Ensure the indices in each SrcVT sized piece are sequential and that
3471  // the same source is used for the whole piece.
3472  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3473  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3474  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3475  IsConcat = false;
3476  break;
3477  }
3478  // Remember which source this index came from.
3479  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3480  }
3481 
3482  // The shuffle is concatenating multiple vectors together. Just emit
3483  // a CONCAT_VECTORS operation.
3484  if (IsConcat) {
3485  SmallVector<SDValue, 8> ConcatOps;
3486  for (auto Src : ConcatSrcs) {
3487  if (Src < 0)
3488  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3489  else if (Src == 0)
3490  ConcatOps.push_back(Src1);
3491  else
3492  ConcatOps.push_back(Src2);
3493  }
3494  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3495  return;
3496  }
3497  }
3498 
3499  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3500  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3501  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3502  PaddedMaskNumElts);
3503 
3504  // Pad both vectors with undefs to make them the same length as the mask.
3505  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3506 
3507  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3508  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3509  MOps1[0] = Src1;
3510  MOps2[0] = Src2;
3511 
3512  Src1 = Src1.isUndef()
3513  ? DAG.getUNDEF(PaddedVT)
3514  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3515  Src2 = Src2.isUndef()
3516  ? DAG.getUNDEF(PaddedVT)
3517  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3518 
3519  // Readjust mask for new input vector length.
3520  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3521  for (unsigned i = 0; i != MaskNumElts; ++i) {
3522  int Idx = Mask[i];
3523  if (Idx >= (int)SrcNumElts)
3524  Idx -= SrcNumElts - PaddedMaskNumElts;
3525  MappedOps[i] = Idx;
3526  }
3527 
3528  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3529 
3530  // If the concatenated vector was padded, extract a subvector with the
3531  // correct number of elements.
3532  if (MaskNumElts != PaddedMaskNumElts)
3533  Result = DAG.getNode(
3534  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3535  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3536 
3537  setValue(&I, Result);
3538  return;
3539  }
3540 
3541  if (SrcNumElts > MaskNumElts) {
3542  // Analyze the access pattern of the vector to see if we can extract
3543  // two subvectors and do the shuffle.
3544  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3545  bool CanExtract = true;
3546  for (int Idx : Mask) {
3547  unsigned Input = 0;
3548  if (Idx < 0)
3549  continue;
3550 
3551  if (Idx >= (int)SrcNumElts) {
3552  Input = 1;
3553  Idx -= SrcNumElts;
3554  }
3555 
3556  // If all the indices come from the same MaskNumElts sized portion of
3557  // the sources we can use extract. Also make sure the extract wouldn't
3558  // extract past the end of the source.
3559  int NewStartIdx = alignDown(Idx, MaskNumElts);
3560  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3561  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3562  CanExtract = false;
3563  // Make sure we always update StartIdx as we use it to track if all
3564  // elements are undef.
3565  StartIdx[Input] = NewStartIdx;
3566  }
3567 
3568  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3569  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3570  return;
3571  }
3572  if (CanExtract) {
3573  // Extract appropriate subvector and generate a vector shuffle
3574  for (unsigned Input = 0; Input < 2; ++Input) {
3575  SDValue &Src = Input == 0 ? Src1 : Src2;
3576  if (StartIdx[Input] < 0)
3577  Src = DAG.getUNDEF(VT);
3578  else {
3579  Src = DAG.getNode(
3580  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3581  DAG.getConstant(StartIdx[Input], DL,
3582  TLI.getVectorIdxTy(DAG.getDataLayout())));
3583  }
3584  }
3585 
3586  // Calculate new mask.
3587  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3588  for (int &Idx : MappedOps) {
3589  if (Idx >= (int)SrcNumElts)
3590  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3591  else if (Idx >= 0)
3592  Idx -= StartIdx[0];
3593  }
3594 
3595  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3596  return;
3597  }
3598  }
3599 
3600  // We can't use either concat vectors or extract subvectors so fall back to
3601  // replacing the shuffle with extract and build vector.
3602  // to insert and build vector.
3603  EVT EltVT = VT.getVectorElementType();
3604  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3606  for (int Idx : Mask) {
3607  SDValue Res;
3608 
3609  if (Idx < 0) {
3610  Res = DAG.getUNDEF(EltVT);
3611  } else {
3612  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3613  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3614 
3615  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3616  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3617  }
3618 
3619  Ops.push_back(Res);
3620  }
3621 
3622  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3623 }
3624 
3625 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3626  ArrayRef<unsigned> Indices;
3627  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3628  Indices = IV->getIndices();
3629  else
3630  Indices = cast<ConstantExpr>(&I)->getIndices();
3631 
3632  const Value *Op0 = I.getOperand(0);
3633  const Value *Op1 = I.getOperand(1);
3634  Type *AggTy = I.getType();
3635  Type *ValTy = Op1->getType();
3636  bool IntoUndef = isa<UndefValue>(Op0);
3637  bool FromUndef = isa<UndefValue>(Op1);
3638 
3639  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3640 
3641  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3642  SmallVector<EVT, 4> AggValueVTs;
3643  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3644  SmallVector<EVT, 4> ValValueVTs;
3645  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3646 
3647  unsigned NumAggValues = AggValueVTs.size();
3648  unsigned NumValValues = ValValueVTs.size();
3649  SmallVector<SDValue, 4> Values(NumAggValues);
3650 
3651  // Ignore an insertvalue that produces an empty object
3652  if (!NumAggValues) {
3653  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3654  return;
3655  }
3656 
3657  SDValue Agg = getValue(Op0);
3658  unsigned i = 0;
3659  // Copy the beginning value(s) from the original aggregate.
3660  for (; i != LinearIndex; ++i)
3661  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3662  SDValue(Agg.getNode(), Agg.getResNo() + i);
3663  // Copy values from the inserted value(s).
3664  if (NumValValues) {
3665  SDValue Val = getValue(Op1);
3666  for (; i != LinearIndex + NumValValues; ++i)
3667  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3668  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3669  }
3670  // Copy remaining value(s) from the original aggregate.
3671  for (; i != NumAggValues; ++i)
3672  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3673  SDValue(Agg.getNode(), Agg.getResNo() + i);
3674 
3675  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3676  DAG.getVTList(AggValueVTs), Values));
3677 }
3678 
3679 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3680  ArrayRef<unsigned> Indices;
3681  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3682  Indices = EV->getIndices();
3683  else
3684  Indices = cast<ConstantExpr>(&I)->getIndices();
3685 
3686  const Value *Op0 = I.getOperand(0);
3687  Type *AggTy = Op0->getType();
3688  Type *ValTy = I.getType();
3689  bool OutOfUndef = isa<UndefValue>(Op0);
3690 
3691  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3692 
3693  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3694  SmallVector<EVT, 4> ValValueVTs;
3695  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3696 
3697  unsigned NumValValues = ValValueVTs.size();
3698 
3699  // Ignore a extractvalue that produces an empty object
3700  if (!NumValValues) {
3701  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3702  return;
3703  }
3704 
3705  SmallVector<SDValue, 4> Values(NumValValues);
3706 
3707  SDValue Agg = getValue(Op0);
3708  // Copy out the selected value(s).
3709  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3710  Values[i - LinearIndex] =
3711  OutOfUndef ?
3712  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3713  SDValue(Agg.getNode(), Agg.getResNo() + i);
3714 
3715  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3716  DAG.getVTList(ValValueVTs), Values));
3717 }
3718 
3719 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3720  Value *Op0 = I.getOperand(0);
3721  // Note that the pointer operand may be a vector of pointers. Take the scalar
3722  // element which holds a pointer.
3723  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3724  SDValue N = getValue(Op0);
3725  SDLoc dl = getCurSDLoc();
3726 
3727  // Normalize Vector GEP - all scalar operands should be converted to the
3728  // splat vector.
3729  unsigned VectorWidth = I.getType()->isVectorTy() ?
3730  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3731 
3732  if (VectorWidth && !N.getValueType().isVector()) {
3733  LLVMContext &Context = *DAG.getContext();
3734  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3735  N = DAG.getSplatBuildVector(VT, dl, N);
3736  }
3737 
3738  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3739  GTI != E; ++GTI) {
3740  const Value *Idx = GTI.getOperand();
3741  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3742  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3743  if (Field) {
3744  // N = N + Offset
3745  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3746 
3747  // In an inbounds GEP with an offset that is nonnegative even when
3748  // interpreted as signed, assume there is no unsigned overflow.
3749  SDNodeFlags Flags;
3750  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3751  Flags.setNoUnsignedWrap(true);
3752 
3753  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3754  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3755  }
3756  } else {
3757  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3758  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3759  APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3760 
3761  // If this is a scalar constant or a splat vector of constants,
3762  // handle it quickly.
3763  const auto *CI = dyn_cast<ConstantInt>(Idx);
3764  if (!CI && isa<ConstantDataVector>(Idx) &&
3765  cast<ConstantDataVector>(Idx)->getSplatValue())
3766  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3767 
3768  if (CI) {
3769  if (CI->isZero())
3770  continue;
3771  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3772  LLVMContext &Context = *DAG.getContext();
3773  SDValue OffsVal = VectorWidth ?
3774  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3775  DAG.getConstant(Offs, dl, IdxTy);
3776 
3777  // In an inbouds GEP with an offset that is nonnegative even when
3778  // interpreted as signed, assume there is no unsigned overflow.
3779  SDNodeFlags Flags;
3780  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3781  Flags.setNoUnsignedWrap(true);
3782 
3783  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3784  continue;
3785  }
3786 
3787  // N = N + Idx * ElementSize;
3788  SDValue IdxN = getValue(Idx);
3789 
3790  if (!IdxN.getValueType().isVector() && VectorWidth) {
3791  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3792  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3793  }
3794 
3795  // If the index is smaller or larger than intptr_t, truncate or extend
3796  // it.
3797  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3798 
3799  // If this is a multiply by a power of two, turn it into a shl
3800  // immediately. This is a very common case.
3801  if (ElementSize != 1) {
3802  if (ElementSize.isPowerOf2()) {
3803  unsigned Amt = ElementSize.logBase2();
3804  IdxN = DAG.getNode(ISD::SHL, dl,
3805  N.getValueType(), IdxN,
3806  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3807  } else {
3808  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3809  IdxN = DAG.getNode(ISD::MUL, dl,
3810  N.getValueType(), IdxN, Scale);
3811  }
3812  }
3813 
3814  N = DAG.getNode(ISD::ADD, dl,
3815  N.getValueType(), N, IdxN);
3816  }
3817  }
3818 
3819  setValue(&I, N);
3820 }
3821 
3822 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3823  // If this is a fixed sized alloca in the entry block of the function,
3824  // allocate it statically on the stack.
3825  if (FuncInfo.StaticAllocaMap.count(&I))
3826  return; // getValue will auto-populate this.
3827 
3828  SDLoc dl = getCurSDLoc();
3829  Type *Ty = I.getAllocatedType();
3830  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3831  auto &DL = DAG.getDataLayout();
3832  uint64_t TySize = DL.getTypeAllocSize(Ty);
3833  unsigned Align =
3834  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3835 
3836  SDValue AllocSize = getValue(I.getArraySize());
3837 
3838  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3839  if (AllocSize.getValueType() != IntPtr)
3840  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3841 
3842  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3843  AllocSize,
3844  DAG.getConstant(TySize, dl, IntPtr));
3845 
3846  // Handle alignment. If the requested alignment is less than or equal to
3847  // the stack alignment, ignore it. If the size is greater than or equal to
3848  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3849  unsigned StackAlign =
3851  if (Align <= StackAlign)
3852  Align = 0;
3853 
3854  // Round the size of the allocation up to the stack alignment size
3855  // by add SA-1 to the size. This doesn't overflow because we're computing
3856  // an address inside an alloca.
3857  SDNodeFlags Flags;
3858  Flags.setNoUnsignedWrap(true);
3859  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3860  DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3861 
3862  // Mask out the low bits for alignment purposes.
3863  AllocSize =
3864  DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3865  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3866 
3867  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3868  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3869  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3870  setValue(&I, DSA);
3871  DAG.setRoot(DSA.getValue(1));
3872 
3873  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3874 }
3875 
3876 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3877  if (I.isAtomic())
3878  return visitAtomicLoad(I);
3879 
3880  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3881  const Value *SV = I.getOperand(0);
3882  if (TLI.supportSwiftError()) {
3883  // Swifterror values can come from either a function parameter with
3884  // swifterror attribute or an alloca with swifterror attribute.
3885  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3886  if (Arg->hasSwiftErrorAttr())
3887  return visitLoadFromSwiftError(I);
3888  }
3889 
3890  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3891  if (Alloca->isSwiftError())
3892  return visitLoadFromSwiftError(I);
3893  }
3894  }
3895 
3896  SDValue Ptr = getValue(SV);
3897 
3898  Type *Ty = I.getType();
3899 
3900  bool isVolatile = I.isVolatile();
3901  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3902  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3903  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3904  unsigned Alignment = I.getAlignment();
3905 
3906  AAMDNodes AAInfo;
3907  I.getAAMetadata(AAInfo);
3908  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3909 
3912  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3913  unsigned NumValues = ValueVTs.size();
3914  if (NumValues == 0)
3915  return;
3916 
3917  SDValue Root;
3918  bool ConstantMemory = false;
3919  if (isVolatile || NumValues > MaxParallelChains)
3920  // Serialize volatile loads with other side effects.
3921  Root = getRoot();
3922  else if (AA &&
3923  AA->pointsToConstantMemory(MemoryLocation(
3924  SV,
3926  AAInfo))) {
3927  // Do not serialize (non-volatile) loads of constant memory with anything.
3928  Root = DAG.getEntryNode();
3929  ConstantMemory = true;
3930  } else {
3931  // Do not serialize non-volatile loads against each other.
3932  Root = DAG.getRoot();
3933  }
3934 
3935  SDLoc dl = getCurSDLoc();
3936 
3937  if (isVolatile)
3938  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3939 
3940  // An aggregate load cannot wrap around the address space, so offsets to its
3941  // parts don't wrap either.
3942  SDNodeFlags Flags;
3943  Flags.setNoUnsignedWrap(true);
3944 
3945  SmallVector<SDValue, 4> Values(NumValues);
3946  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3947  EVT PtrVT = Ptr.getValueType();
3948  unsigned ChainI = 0;
3949  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3950  // Serializing loads here may result in excessive register pressure, and
3951  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3952  // could recover a bit by hoisting nodes upward in the chain by recognizing
3953  // they are side-effect free or do not alias. The optimizer should really
3954  // avoid this case by converting large object/array copies to llvm.memcpy
3955  // (MaxParallelChains should always remain as failsafe).
3956  if (ChainI == MaxParallelChains) {
3957  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3958  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3959  makeArrayRef(Chains.data(), ChainI));
3960  Root = Chain;
3961  ChainI = 0;
3962  }
3963  SDValue A = DAG.getNode(ISD::ADD, dl,
3964  PtrVT, Ptr,
3965  DAG.getConstant(Offsets[i], dl, PtrVT),
3966  Flags);
3967  auto MMOFlags = MachineMemOperand::MONone;
3968  if (isVolatile)
3969  MMOFlags |= MachineMemOperand::MOVolatile;
3970  if (isNonTemporal)
3972  if (isInvariant)
3973  MMOFlags |= MachineMemOperand::MOInvariant;
3974  if (isDereferenceable)
3976  MMOFlags |= TLI.getMMOFlags(I);
3977 
3978  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3979  MachinePointerInfo(SV, Offsets[i]), Alignment,
3980  MMOFlags, AAInfo, Ranges);
3981 
3982  Values[i] = L;
3983  Chains[ChainI] = L.getValue(1);
3984  }
3985 
3986  if (!ConstantMemory) {
3987  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3988  makeArrayRef(Chains.data(), ChainI));
3989  if (isVolatile)
3990  DAG.setRoot(Chain);
3991  else
3992  PendingLoads.push_back(Chain);
3993  }
3994 
3995  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3996  DAG.getVTList(ValueVTs), Values));
3997 }
3998 
3999 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4001  "call visitStoreToSwiftError when backend supports swifterror");
4002 
4005  const Value *SrcV = I.getOperand(0);
4007  SrcV->getType(), ValueVTs, &Offsets);
4008  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4009  "expect a single EVT for swifterror");
4010 
4011  SDValue Src = getValue(SrcV);
4012  // Create a virtual register, then update the virtual register.
4013  unsigned VReg; bool CreatedVReg;
4014  std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
4015  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4016  // Chain can be getRoot or getControlRoot.
4017  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4018  SDValue(Src.getNode(), Src.getResNo()));
4019  DAG.setRoot(CopyNode);
4020  if (CreatedVReg)
4021  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
4022 }
4023 
4024 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4026  "call visitLoadFromSwiftError when backend supports swifterror");
4027 
4028  assert(!I.isVolatile() &&
4029  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
4031  "Support volatile, non temporal, invariant for load_from_swift_error");
4032 
4033  const Value *SV = I.getOperand(0);
4034  Type *Ty = I.getType();
4035  AAMDNodes AAInfo;
4036  I.getAAMetadata(AAInfo);
4037  assert(
4038  (!AA ||
4039  !AA->pointsToConstantMemory(MemoryLocation(
4041  AAInfo))) &&
4042  "load_from_swift_error should not be constant memory");
4043 
4047  ValueVTs, &Offsets);
4048  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4049  "expect a single EVT for swifterror");
4050 
4051  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4052  SDValue L = DAG.getCopyFromReg(
4053  getRoot(), getCurSDLoc(),
4054  FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
4055  ValueVTs[0]);
4056 
4057  setValue(&I, L);
4058 }
4059 
4060 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4061  if (I.isAtomic())
4062  return visitAtomicStore(I);
4063 
4064  const Value *SrcV = I.getOperand(0);
4065  const Value *PtrV = I.getOperand(1);
4066 
4067  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4068  if (TLI.supportSwiftError()) {
4069  // Swifterror values can come from either a function parameter with
4070  // swifterror attribute or an alloca with swifterror attribute.
4071  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4072  if (Arg->hasSwiftErrorAttr())
4073  return visitStoreToSwiftError(I);
4074  }
4075 
4076  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4077  if (Alloca->isSwiftError())
4078  return visitStoreToSwiftError(I);
4079  }
4080  }
4081 
4085  SrcV->getType(), ValueVTs, &Offsets);
4086  unsigned NumValues = ValueVTs.size();
4087  if (NumValues == 0)
4088  return;
4089 
4090  // Get the lowered operands. Note that we do this after
4091  // checking if NumResults is zero, because with zero results
4092  // the operands won't have values in the map.
4093  SDValue Src = getValue(SrcV);
4094  SDValue Ptr = getValue(PtrV);
4095 
4096  SDValue Root = getRoot();
4097  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4098  SDLoc dl = getCurSDLoc();
4099  EVT PtrVT = Ptr.getValueType();
4100  unsigned Alignment = I.getAlignment();
4101  AAMDNodes AAInfo;
4102  I.getAAMetadata(AAInfo);
4103 
4104  auto MMOFlags = MachineMemOperand::MONone;
4105  if (I.isVolatile())
4106  MMOFlags |= MachineMemOperand::MOVolatile;
4107  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
4109  MMOFlags |= TLI.getMMOFlags(I);
4110 
4111  // An aggregate load cannot wrap around the address space, so offsets to its
4112  // parts don't wrap either.
4113  SDNodeFlags Flags;
4114  Flags.setNoUnsignedWrap(true);
4115 
4116  unsigned ChainI = 0;
4117  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4118  // See visitLoad comments.
4119  if (ChainI == MaxParallelChains) {
4120  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4121  makeArrayRef(Chains.data(), ChainI));
4122  Root = Chain;
4123  ChainI = 0;
4124  }
4125  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
4126  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
4127  SDValue St = DAG.getStore(
4128  Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
4129  MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
4130  Chains[ChainI] = St;
4131  }
4132 
4133  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4134  makeArrayRef(Chains.data(), ChainI));
4135  DAG.setRoot(StoreNode);
4136 }
4137 
4138 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4139  bool IsCompressing) {
4140  SDLoc sdl = getCurSDLoc();
4141 
4142  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4143  unsigned& Alignment) {
4144  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4145  Src0 = I.getArgOperand(0);
4146  Ptr = I.getArgOperand(1);
4147  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
4148  Mask = I.getArgOperand(3);
4149  };
4150  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4151  unsigned& Alignment) {
4152  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4153  Src0 = I.getArgOperand(0);
4154  Ptr = I.getArgOperand(1);
4155  Mask = I.getArgOperand(2);
4156  Alignment = 0;
4157  };
4158 
4159  Value *PtrOperand, *MaskOperand, *Src0Operand;
4160  unsigned Alignment;
4161  if (IsCompressing)
4162  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4163  else
4164  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4165 
4166  SDValue Ptr = getValue(PtrOperand);
4167  SDValue Src0 = getValue(Src0Operand);
4168  SDValue Mask = getValue(MaskOperand);
4169 
4170  EVT VT = Src0.getValueType();
4171  if (!Alignment)
4172  Alignment = DAG.getEVTAlignment(VT);
4173 
4174  AAMDNodes AAInfo;
4175  I.getAAMetadata(AAInfo);
4176 
4177  MachineMemOperand *MMO =
4178  DAG.getMachineFunction().
4179  getMachineMemOperand(MachinePointerInfo(PtrOperand),
4181  Alignment, AAInfo);
4182  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
4183  MMO, false /* Truncating */,
4184  IsCompressing);
4185  DAG.setRoot(StoreNode);
4186  setValue(&I, StoreNode);
4187 }
4188 
4189 // Get a uniform base for the Gather/Scatter intrinsic.
4190 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4191 // We try to represent it as a base pointer + vector of indices.
4192 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4193 // The first operand of the GEP may be a single pointer or a vector of pointers
4194 // Example:
4195 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4196 // or
4197 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4198 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4199 //
4200 // When the first GEP operand is a single pointer - it is the uniform base we
4201 // are looking for. If first operand of the GEP is a splat vector - we
4202 // extract the splat value and use it as a uniform base.
4203 // In all other cases the function returns 'false'.
4204 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
4205  SDValue &Scale, SelectionDAGBuilder* SDB) {
4206  SelectionDAG& DAG = SDB->DAG;
4207  LLVMContext &Context = *DAG.getContext();
4208 
4209  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
4211  if (!GEP)
4212  return false;
4213 
4214  const Value *GEPPtr = GEP->getPointerOperand();
4215  if (!GEPPtr->getType()->isVectorTy())
4216  Ptr = GEPPtr;
4217  else if (!(Ptr = getSplatValue(GEPPtr)))
4218  return false;
4219 
4220  unsigned FinalIndex = GEP->getNumOperands() - 1;
4221  Value *IndexVal = GEP->getOperand(FinalIndex);
4222 
4223  // Ensure all the other indices are 0.
4224  for (unsigned i = 1; i < FinalIndex; ++i) {
4225  auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
4226  if (!C || !C->isZero())
4227  return false;
4228  }
4229 
4230  // The operands of the GEP may be defined in another basic block.
4231  // In this case we'll not find nodes for the operands.
4232  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
4233  return false;
4234 
4235  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4236  const DataLayout &DL = DAG.getDataLayout();
4237  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4238  SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4239  Base = SDB->getValue(Ptr);
4240  Index = SDB->getValue(IndexVal);
4241 
4242  if (!Index.getValueType().isVector()) {
4243  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4244  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4245  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4246  }
4247  return true;
4248 }
4249 
4250 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4251  SDLoc sdl = getCurSDLoc();
4252 
4253  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4254  const Value *Ptr = I.getArgOperand(1);
4255  SDValue Src0 = getValue(I.getArgOperand(0));
4256  SDValue Mask = getValue(I.getArgOperand(3));
4257  EVT VT = Src0.getValueType();
4258  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4259  if (!Alignment)
4260  Alignment = DAG.getEVTAlignment(VT);
4261  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4262 
4263  AAMDNodes AAInfo;
4264  I.getAAMetadata(AAInfo);
4265 
4266  SDValue Base;
4267  SDValue Index;
4268  SDValue Scale;
4269  const Value *BasePtr = Ptr;
4270  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4271 
4272  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4274  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4275  MachineMemOperand::MOStore, VT.getStoreSize(),
4276  Alignment, AAInfo);
4277  if (!UniformBase) {
4278  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4279  Index = getValue(Ptr);
4280  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4281  }
4282  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4283  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4284  Ops, MMO);
4285  DAG.setRoot(Scatter);
4286  setValue(&I, Scatter);
4287 }
4288 
4289 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4290  SDLoc sdl = getCurSDLoc();
4291 
4292  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4293  unsigned& Alignment) {
4294  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4295  Ptr = I.getArgOperand(0);
4296  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4297  Mask = I.getArgOperand(2);
4298  Src0 = I.getArgOperand(3);
4299  };
4300  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4301  unsigned& Alignment) {
4302  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4303  Ptr = I.getArgOperand(0);
4304  Alignment = 0;
4305  Mask = I.getArgOperand(1);
4306  Src0 = I.getArgOperand(2);
4307  };
4308 
4309  Value *PtrOperand, *MaskOperand, *Src0Operand;
4310  unsigned Alignment;
4311  if (IsExpanding)
4312  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4313  else
4314  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4315 
4316  SDValue Ptr = getValue(PtrOperand);
4317  SDValue Src0 = getValue(Src0Operand);
4318  SDValue Mask = getValue(MaskOperand);
4319 
4320  EVT VT = Src0.getValueType();
4321  if (!Alignment)
4322  Alignment = DAG.getEVTAlignment(VT);
4323 
4324  AAMDNodes AAInfo;
4325  I.getAAMetadata(AAInfo);
4326  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4327 
4328  // Do not serialize masked loads of constant memory with anything.
4329  bool AddToChain =
4330  !AA || !AA->pointsToConstantMemory(MemoryLocation(
4331  PtrOperand,
4334  AAInfo));
4335  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4336 
4337  MachineMemOperand *MMO =
4338  DAG.getMachineFunction().
4339  getMachineMemOperand(MachinePointerInfo(PtrOperand),
4341  Alignment, AAInfo, Ranges);
4342 
4343  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4344  ISD::NON_EXTLOAD, IsExpanding);
4345  if (AddToChain)
4346  PendingLoads.push_back(Load.getValue(1));
4347  setValue(&I, Load);
4348 }
4349 
4350 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4351  SDLoc sdl = getCurSDLoc();
4352 
4353  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4354  const Value *Ptr = I.getArgOperand(0);
4355  SDValue Src0 = getValue(I.getArgOperand(3));
4356  SDValue Mask = getValue(I.getArgOperand(2));
4357 
4358  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4359  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4360  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4361  if (!Alignment)
4362  Alignment = DAG.getEVTAlignment(VT);
4363 
4364  AAMDNodes AAInfo;
4365  I.getAAMetadata(AAInfo);
4366  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4367 
4368  SDValue Root = DAG.getRoot();
4369  SDValue Base;
4370  SDValue Index;
4371  SDValue Scale;
4372  const Value *BasePtr = Ptr;
4373  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4374  bool ConstantMemory = false;
4375  if (UniformBase && AA &&
4376  AA->pointsToConstantMemory(
4377  MemoryLocation(BasePtr,
4380  AAInfo))) {
4381  // Do not serialize (non-volatile) loads of constant memory with anything.
4382  Root = DAG.getEntryNode();
4383  ConstantMemory = true;
4384  }
4385 
4386  MachineMemOperand *MMO =
4387  DAG.getMachineFunction().
4388  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4390  Alignment, AAInfo, Ranges);
4391 
4392  if (!UniformBase) {
4393  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4394  Index = getValue(Ptr);
4395  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4396  }
4397  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4398  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4399  Ops, MMO);
4400 
4401  SDValue OutChain = Gather.getValue(1);
4402  if (!ConstantMemory)
4403  PendingLoads.push_back(OutChain);
4404  setValue(&I, Gather);
4405 }
4406 
4407 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4408  SDLoc dl = getCurSDLoc();
4409  AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4410  AtomicOrdering FailureOrdering = I.getFailureOrdering();
4411  SyncScope::ID SSID = I.getSyncScopeID();
4412 
4413  SDValue InChain = getRoot();
4414 
4415  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4416  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4417 
4418  auto Alignment = DAG.getEVTAlignment(MemVT);
4419 
4420  // FIXME: Volatile isn't really correct; we should keep track of atomic
4421  // orderings in the memoperand.
4424 
4425  MachineFunction &MF = DAG.getMachineFunction();
4426  MachineMemOperand *MMO =
4428  Flags, MemVT.getStoreSize(), Alignment,
4429  AAMDNodes(), nullptr, SSID, SuccessOrdering,
4430  FailureOrdering);
4431 
4433  dl, MemVT, VTs, InChain,
4434  getValue(I.getPointerOperand()),
4435  getValue(I.getCompareOperand()),
4436  getValue(I.getNewValOperand()), MMO);
4437 
4438  SDValue OutChain = L.getValue(2);
4439 
4440  setValue(&I, L);
4441  DAG.setRoot(OutChain);
4442 }
4443 
4444 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4445  SDLoc dl = getCurSDLoc();
4446  ISD::NodeType NT;
4447  switch (I.getOperation()) {
4448  default: llvm_unreachable("Unknown atomicrmw operation");
4449  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4450  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4451  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4452  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4453  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4454  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4455  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4456  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4457  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4458  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4459  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4460  case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4461  case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4462  }
4463  AtomicOrdering Ordering = I.getOrdering();
4464  SyncScope::ID SSID = I.getSyncScopeID();
4465 
4466  SDValue InChain = getRoot();
4467 
4468  auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4469  auto Alignment = DAG.getEVTAlignment(MemVT);
4470 
4471  // For now, atomics are considered to be volatile always, and they are
4472  // chained as such.
4473  // FIXME: Volatile isn't really correct; we should keep track of atomic
4474  // orderings in the memoperand.
4475  auto Flags = MachineMemOperand::MOVolatile |
4477 
4478  MachineFunction &MF = DAG.getMachineFunction();
4479  MachineMemOperand *MMO =
4481  MemVT.getStoreSize(), Alignment, AAMDNodes(),
4482  nullptr, SSID, Ordering);
4483 
4484  SDValue L =
4485  DAG.getAtomic(NT, dl, MemVT, InChain,
4486  getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4487  MMO);
4488 
4489  SDValue OutChain = L.getValue(1);
4490 
4491  setValue(&I, L);
4492  DAG.setRoot(OutChain);
4493 }
4494 
4495 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4496  SDLoc dl = getCurSDLoc();
4497  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4498  SDValue Ops[3];
4499  Ops[0] = getRoot();
4500  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4501  TLI.getFenceOperandTy(DAG.getDataLayout()));
4502  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4503  TLI.getFenceOperandTy(DAG.getDataLayout()));
4504  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4505 }
4506 
4507 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4508  SDLoc dl = getCurSDLoc();
4509  AtomicOrdering Order = I.getOrdering();
4510  SyncScope::ID SSID = I.getSyncScopeID();
4511 
4512  SDValue InChain = getRoot();
4513 
4514  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4515  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4516 
4517  if (!TLI.supportsUnalignedAtomics() &&
4518  I.getAlignment() < VT.getStoreSize())
4519  report_fatal_error("Cannot generate unaligned atomic load");
4520 
4521  MachineMemOperand *MMO =
4522  DAG.getMachineFunction().
4523  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4526  VT.getStoreSize(),
4527  I.getAlignment() ? I.getAlignment() :
4528  DAG.getEVTAlignment(VT),
4529  AAMDNodes(), nullptr, SSID, Order);
4530 
4531  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4532  SDValue L =
4533  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4534  getValue(I.getPointerOperand()), MMO);
4535 
4536  SDValue OutChain = L.getValue(1);
4537 
4538  setValue(&I, L);
4539  DAG.setRoot(OutChain);
4540 }
4541 
4542 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4543  SDLoc dl = getCurSDLoc();
4544 
4545  AtomicOrdering Ordering = I.getOrdering();
4546  SyncScope::ID SSID = I.getSyncScopeID();
4547 
4548  SDValue InChain = getRoot();
4549 
4550  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4551  EVT VT =
4553 
4554  if (I.getAlignment() < VT.getStoreSize())
4555  report_fatal_error("Cannot generate unaligned atomic store");
4556 
4557  // For now, atomics are considered to be volatile always, and they are
4558  // chained as such.
4559  // FIXME: Volatile isn't really correct; we should keep track of atomic
4560  // orderings in the memoperand.
4562 
4563  MachineFunction &MF = DAG.getMachineFunction();
4564  MachineMemOperand *MMO =
4566  VT.getStoreSize(), I.getAlignment(), AAMDNodes(),
4567  nullptr, SSID, Ordering);
4568  SDValue OutChain =
4569  DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, InChain,
4570  getValue(I.getPointerOperand()), getValue(I.getValueOperand()),
4571  MMO);
4572 
4573 
4574  DAG.setRoot(OutChain);
4575 }
4576 
4577 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4578 /// node.
4579 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4580  unsigned Intrinsic) {
4581  // Ignore the callsite's attributes. A specific call site may be marked with
4582  // readnone, but the lowering code will expect the chain based on the
4583  // definition.
4584  const Function *F = I.getCalledFunction();
4585  bool HasChain = !F->doesNotAccessMemory();
4586  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4587 
4588  // Build the operand list.
4590  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4591  if (OnlyLoad) {
4592  // We don't need to serialize loads against other loads.
4593  Ops.push_back(DAG.getRoot());
4594  } else {
4595  Ops.push_back(getRoot());
4596  }
4597  }
4598 
4599  // Info is set by getTgtMemInstrinsic
4600  TargetLowering::IntrinsicInfo Info;
4601  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4602  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4603  DAG.getMachineFunction(),
4604  Intrinsic);
4605 
4606  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4607  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4608  Info.opc == ISD::INTRINSIC_W_CHAIN)
4609  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4610  TLI.getPointerTy(DAG.getDataLayout())));
4611 
4612  // Add all operands of the call to the operand list.
4613  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4614  SDValue Op = getValue(I.getArgOperand(i));
4615  Ops.push_back(Op);
4616  }
4617 
4619  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4620 
4621  if (HasChain)
4622  ValueVTs.push_back(MVT::Other);
4623 
4624  SDVTList VTs = DAG.getVTList(ValueVTs);
4625 
4626  // Create the node.
4627  SDValue Result;
4628  if (IsTgtIntrinsic) {
4629  // This is target intrinsic that touches memory
4630  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4631  Ops, Info.memVT,
4632  MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4633  Info.flags, Info.size);
4634  } else if (!HasChain) {
4635  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4636  } else if (!I.getType()->isVoidTy()) {
4637  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4638  } else {
4639  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4640  }
4641 
4642  if (HasChain) {
4643  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4644  if (OnlyLoad)
4645  PendingLoads.push_back(Chain);
4646  else
4647  DAG.setRoot(Chain);
4648  }
4649 
4650  if (!I.getType()->isVoidTy()) {
4651  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4652  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4653  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4654  } else
4655  Result = lowerRangeToAssertZExt(DAG, I, Result);
4656 
4657  setValue(&I, Result);
4658  }
4659 }
4660 
4661 /// GetSignificand - Get the significand and build it into a floating-point
4662 /// number with exponent of 1:
4663 ///
4664 /// Op = (Op & 0x007fffff) | 0x3f800000;
4665 ///
4666 /// where Op is the hexadecimal representation of floating point value.
4668  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4669  DAG.getConstant(0x007fffff, dl, MVT::i32));
4670  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4671  DAG.getConstant(0x3f800000, dl, MVT::i32));
4672  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4673 }
4674 
4675 /// GetExponent - Get the exponent:
4676 ///
4677 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4678 ///
4679 /// where Op is the hexadecimal representation of floating point value.
4681  const TargetLowering &TLI, const SDLoc &dl) {
4682  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4683  DAG.getConstant(0x7f800000, dl, MVT::i32));
4684  SDValue t1 = DAG.getNode(
4685  ISD::SRL, dl, MVT::i32, t0,
4686  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4687  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4688  DAG.getConstant(127, dl, MVT::i32));
4689  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4690 }
4691 
4692 /// getF32Constant - Get 32-bit floating point constant.
4693 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4694  const SDLoc &dl) {
4695  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4696  MVT::f32);
4697 }
4698 
4700  SelectionDAG &DAG) {
4701  // TODO: What fast-math-flags should be set on the floating-point nodes?
4702 
4703  // IntegerPartOfX = ((int32_t)(t0);
4704  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4705 
4706  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4707  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4708  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4709 
4710  // IntegerPartOfX <<= 23;
4711  IntegerPartOfX = DAG.getNode(
4712  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4714  DAG.getDataLayout())));
4715 
4716  SDValue TwoToFractionalPartOfX;
4717  if (LimitFloatPrecision <= 6) {
4718  // For floating-point precision of 6:
4719  //
4720  // TwoToFractionalPartOfX =
4721  // 0.997535578f +
4722  // (0.735607626f + 0.252464424f * x) * x;
4723  //
4724  // error 0.0144103317, which is 6 bits
4725  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4726  getF32Constant(DAG, 0x3e814304, dl));
4727  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4728  getF32Constant(DAG, 0x3f3c50c8, dl));
4729  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4730  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4731  getF32Constant(DAG, 0x3f7f5e7e, dl));
4732  } else if (LimitFloatPrecision <= 12) {
4733  // For floating-point precision of 12:
4734  //
4735  // TwoToFractionalPartOfX =
4736  // 0.999892986f +
4737  // (0.696457318f +
4738  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4739  //
4740  // error 0.000107046256, which is 13 to 14 bits
4741  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4742  getF32Constant(DAG, 0x3da235e3, dl));
4743  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4744  getF32Constant(DAG, 0x3e65b8f3, dl));
4745  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4746  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4747  getF32Constant(DAG, 0x3f324b07, dl));
4748  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4749  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4750  getF32Constant(DAG, 0x3f7ff8fd, dl));
4751  } else { // LimitFloatPrecision <= 18
4752  // For floating-point precision of 18:
4753  //
4754  // TwoToFractionalPartOfX =
4755  // 0.999999982f +
4756  // (0.693148872f +
4757  // (0.240227044f +
4758  // (0.554906021e-1f +
4759  // (0.961591928e-2f +
4760  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4761  // error 2.47208000*10^(-7), which is better than 18 bits
4762  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4763  getF32Constant(DAG, 0x3924b03e, dl));
4764  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4765  getF32Constant(DAG, 0x3ab24b87, dl));
4766  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4767  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4768  getF32Constant(DAG, 0x3c1d8c17, dl));
4769  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4770  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4771  getF32Constant(DAG, 0x3d634a1d, dl));
4772  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4773  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4774  getF32Constant(DAG, 0x3e75fe14, dl));
4775  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4776  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4777  getF32Constant(DAG, 0x3f317234, dl));
4778  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4779  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4780  getF32Constant(DAG, 0x3f800000, dl));
4781  }
4782 
4783  // Add the exponent into the result in integer domain.
4784  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4785  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4786  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4787 }
4788 
4789 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4790 /// limited-precision mode.
4791 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4792  const TargetLowering &TLI) {
4793  if (Op.getValueType() == MVT::f32 &&
4794  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4795 
4796  // Put the exponent in the right bit position for later addition to the
4797  // final result:
4798  //
4799  // #define LOG2OFe 1.4426950f
4800  // t0 = Op * LOG2OFe
4801 
4802  // TODO: What fast-math-flags should be set here?
4803  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4804  getF32Constant(DAG, 0x3fb8aa3b, dl));
4805  return getLimitedPrecisionExp2(t0, dl, DAG);
4806  }
4807 
4808  // No special expansion.
4809  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4810 }
4811 
4812 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4813 /// limited-precision mode.
4814 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4815  const TargetLowering &TLI) {
4816  // TODO: What fast-math-flags should be set on the floating-point nodes?
4817 
4818  if (Op.getValueType() == MVT::f32 &&
4819  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4820  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4821 
4822  // Scale the exponent by log(2) [0.69314718f].
4823  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4824  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4825  getF32Constant(DAG, 0x3f317218, dl));
4826 
4827  // Get the significand and build it into a floating-point number with
4828  // exponent of 1.
4829  SDValue X = GetSignificand(DAG, Op1, dl);
4830 
4831  SDValue LogOfMantissa;
4832  if (LimitFloatPrecision <= 6) {
4833  // For floating-point precision of 6:
4834  //
4835  // LogofMantissa =
4836  // -1.1609546f +
4837  // (1.4034025f - 0.23903021f * x) * x;
4838  //
4839  // error 0.0034276066, which is better than 8 bits
4840  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4841  getF32Constant(DAG, 0xbe74c456, dl));
4842  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4843  getF32Constant(DAG, 0x3fb3a2b1, dl));
4844  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4845  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4846  getF32Constant(DAG, 0x3f949a29, dl));
4847  } else if (LimitFloatPrecision <= 12) {
4848  // For floating-point precision of 12:
4849  //
4850  // LogOfMantissa =
4851  // -1.7417939f +
4852  // (2.8212026f +
4853  // (-1.4699568f +
4854  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4855  //
4856  // error 0.000061011436, which is 14 bits
4857  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4858  getF32Constant(DAG, 0xbd67b6d6, dl));
4859  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4860  getF32Constant(DAG, 0x3ee4f4b8, dl));
4861  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4862  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4863  getF32Constant(DAG, 0x3fbc278b, dl));
4864  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4865  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4866  getF32Constant(DAG, 0x40348e95, dl));
4867  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4868  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4869  getF32Constant(DAG, 0x3fdef31a, dl));
4870  } else { // LimitFloatPrecision <= 18
4871  // For floating-point precision of 18:
4872  //
4873  // LogOfMantissa =
4874  // -2.1072184f +
4875  // (4.2372794f +
4876  // (-3.7029485f +
4877  // (2.2781945f +
4878  // (-0.87823314f +
4879  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4880  //
4881  // error 0.0000023660568, which is better than 18 bits
4882  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4883  getF32Constant(DAG, 0xbc91e5ac, dl));
4884  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4885  getF32Constant(DAG, 0x3e4350aa, dl));
4886  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4887  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4888  getF32Constant(DAG, 0x3f60d3e3, dl));
4889  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4890  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4891  getF32Constant(DAG, 0x4011cdf0, dl));
4892  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4893  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4894  getF32Constant(DAG, 0x406cfd1c, dl));
4895  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4896  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4897  getF32Constant(DAG, 0x408797cb, dl));
4898  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4899  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4900  getF32Constant(DAG, 0x4006dcab, dl));
4901  }
4902 
4903  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4904  }
4905 
4906  // No special expansion.
4907  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4908 }
4909 
4910 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4911 /// limited-precision mode.
4912 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4913  const TargetLowering &TLI) {
4914  // TODO: What fast-math-flags should be set on the floating-point nodes?
4915 
4916  if (Op.getValueType() == MVT::f32 &&
4917  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4918  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4919 
4920  // Get the exponent.
4921  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4922 
4923  // Get the significand and build it into a floating-point number with
4924  // exponent of 1.
4925  SDValue X = GetSignificand(DAG, Op1, dl);
4926 
4927  // Different possible minimax approximations of significand in
4928  // floating-point for various degrees of accuracy over [1,2].
4929  SDValue Log2ofMantissa;
4930  if (LimitFloatPrecision <= 6) {
4931  // For floating-point precision of 6:
4932  //
4933  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4934  //
4935  // error 0.0049451742, which is more than 7 bits
4936  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4937  getF32Constant(DAG, 0xbeb08fe0, dl));
4938  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4939  getF32Constant(DAG, 0x40019463, dl));
4940  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4941  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4942  getF32Constant(DAG, 0x3fd6633d, dl));
4943  } else if (LimitFloatPrecision <= 12) {
4944  // For floating-point precision of 12:
4945  //
4946  // Log2ofMantissa =
4947  // -2.51285454f +
4948  // (4.07009056f +
4949  // (-2.12067489f +
4950  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4951  //
4952  // error 0.0000876136000, which is better than 13 bits
4953  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4954  getF32Constant(DAG, 0xbda7262e, dl));
4955  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4956  getF32Constant(DAG, 0x3f25280b, dl));
4957  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4958  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4959  getF32Constant(DAG, 0x4007b923, dl));
4960  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4961  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4962  getF32Constant(DAG, 0x40823e2f, dl));
4963  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4964  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4965  getF32Constant(DAG, 0x4020d29c, dl));
4966  } else { // LimitFloatPrecision <= 18
4967  // For floating-point precision of 18:
4968  //
4969  // Log2ofMantissa =
4970  // -3.0400495f +
4971  // (6.1129976f +
4972  // (-5.3420409f +
4973  // (3.2865683f +
4974  // (-1.2669343f +
4975  // (0.27515199f -
4976  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4977  //
4978  // error 0.0000018516, which is better than 18 bits
4979  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4980  getF32Constant(DAG, 0xbcd2769e, dl));
4981  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4982  getF32Constant(DAG, 0x3e8ce0b9, dl));
4983  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4984  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4985  getF32Constant(DAG, 0x3fa22ae7, dl));
4986  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4987  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4988  getF32Constant(DAG, 0x40525723, dl));
4989  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4990  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4991  getF32Constant(DAG, 0x40aaf200, dl));
4992  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4993  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4994  getF32Constant(DAG, 0x40c39dad, dl));
4995  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4996  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4997  getF32Constant(DAG, 0x4042902c, dl));
4998  }
4999 
5000  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5001  }
5002 
5003  // No special expansion.
5004  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
5005 }
5006 
5007 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5008 /// limited-precision mode.
5010  const TargetLowering &TLI) {
5011  // TODO: What fast-math-flags should be set on the floating-point nodes?
5012 
5013  if (Op.getValueType() == MVT::f32 &&
5014  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5015  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5016 
5017  // Scale the exponent by log10(2) [0.30102999f].
5018  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5019  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5020  getF32Constant(DAG, 0x3e9a209a, dl));
5021 
5022  // Get the significand and build it into a floating-point number with
5023  // exponent of 1.
5024  SDValue X = GetSignificand(DAG, Op1, dl);
5025 
5026  SDValue Log10ofMantissa;
5027  if (LimitFloatPrecision <= 6) {
5028  // For floating-point precision of 6:
5029  //
5030  // Log10ofMantissa =
5031  // -0.50419619f +
5032  // (0.60948995f - 0.10380950f * x) * x;
5033  //
5034  // error 0.0014886165, which is 6 bits
5035  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5036  getF32Constant(DAG, 0xbdd49a13, dl));
5037  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5038  getF32Constant(DAG, 0x3f1c0789, dl));
5039  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5040  Log10ofMantissa = DAG.getNode(