LLVM  9.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/BitVector.h"
19 #include "llvm/ADT/DenseMap.h"
20 #include "llvm/ADT/None.h"
21 #include "llvm/ADT/Optional.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/ADT/Twine.h"
33 #include "llvm/Analysis/Loads.h"
38 #include "llvm/CodeGen/Analysis.h"
56 #include "llvm/CodeGen/StackMaps.h"
65 #include "llvm/IR/Argument.h"
66 #include "llvm/IR/Attributes.h"
67 #include "llvm/IR/BasicBlock.h"
68 #include "llvm/IR/CFG.h"
69 #include "llvm/IR/CallSite.h"
70 #include "llvm/IR/CallingConv.h"
71 #include "llvm/IR/Constant.h"
72 #include "llvm/IR/ConstantRange.h"
73 #include "llvm/IR/Constants.h"
74 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugLoc.h"
77 #include "llvm/IR/DerivedTypes.h"
78 #include "llvm/IR/Function.h"
80 #include "llvm/IR/InlineAsm.h"
81 #include "llvm/IR/InstrTypes.h"
82 #include "llvm/IR/Instruction.h"
83 #include "llvm/IR/Instructions.h"
84 #include "llvm/IR/IntrinsicInst.h"
85 #include "llvm/IR/Intrinsics.h"
86 #include "llvm/IR/LLVMContext.h"
87 #include "llvm/IR/Metadata.h"
88 #include "llvm/IR/Module.h"
89 #include "llvm/IR/Operator.h"
90 #include "llvm/IR/PatternMatch.h"
91 #include "llvm/IR/Statepoint.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCSymbol.h"
99 #include "llvm/Support/Casting.h"
100 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/Compiler.h"
103 #include "llvm/Support/Debug.h"
106 #include "llvm/Support/MathExtras.h"
112 #include <algorithm>
113 #include <cassert>
114 #include <cstddef>
115 #include <cstdint>
116 #include <cstring>
117 #include <iterator>
118 #include <limits>
119 #include <numeric>
120 #include <tuple>
121 #include <utility>
122 #include <vector>
123 
124 using namespace llvm;
125 using namespace PatternMatch;
126 
127 #define DEBUG_TYPE "isel"
128 
129 /// LimitFloatPrecision - Generate low-precision inline sequences for
130 /// some float libcalls (6, 8 or 12 bits).
131 static unsigned LimitFloatPrecision;
132 
134  LimitFPPrecision("limit-float-precision",
135  cl::desc("Generate low-precision inline sequences "
136  "for some float libcalls"),
137  cl::location(LimitFloatPrecision), cl::Hidden,
138  cl::init(0));
139 
141  "switch-peel-threshold", cl::Hidden, cl::init(66),
142  cl::desc("Set the case probability threshold for peeling the case from a "
143  "switch statement. A value greater than 100 will void this "
144  "optimization"));
145 
146 // Limit the width of DAG chains. This is important in general to prevent
147 // DAG-based analysis from blowing up. For example, alias analysis and
148 // load clustering may not complete in reasonable time. It is difficult to
149 // recognize and avoid this situation within each individual analysis, and
150 // future analyses are likely to have the same behavior. Limiting DAG width is
151 // the safe approach and will be especially important with global DAGs.
152 //
153 // MaxParallelChains default is arbitrarily high to avoid affecting
154 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
155 // sequence over this should have been converted to llvm.memcpy by the
156 // frontend. It is easy to induce this behavior with .ll code such as:
157 // %buffer = alloca [4096 x i8]
158 // %data = load [4096 x i8]* %argPtr
159 // store [4096 x i8] %data, [4096 x i8]* %buffer
160 static const unsigned MaxParallelChains = 64;
161 
162 // Return the calling convention if the Value passed requires ABI mangling as it
163 // is a parameter to a function or a return value from a function which is not
164 // an intrinsic.
166  if (auto *R = dyn_cast<ReturnInst>(V))
167  return R->getParent()->getParent()->getCallingConv();
168 
169  if (auto *CI = dyn_cast<CallInst>(V)) {
170  const bool IsInlineAsm = CI->isInlineAsm();
171  const bool IsIndirectFunctionCall =
172  !IsInlineAsm && !CI->getCalledFunction();
173 
174  // It is possible that the call instruction is an inline asm statement or an
175  // indirect function call in which case the return value of
176  // getCalledFunction() would be nullptr.
177  const bool IsInstrinsicCall =
178  !IsInlineAsm && !IsIndirectFunctionCall &&
179  CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
180 
181  if (!IsInlineAsm && !IsInstrinsicCall)
182  return CI->getCallingConv();
183  }
184 
185  return None;
186 }
187 
188 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
189  const SDValue *Parts, unsigned NumParts,
190  MVT PartVT, EVT ValueVT, const Value *V,
192 
193 /// getCopyFromParts - Create a value that contains the specified legal parts
194 /// combined into the value they represent. If the parts combine to a type
195 /// larger than ValueVT then AssertOp can be used to specify whether the extra
196 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
197 /// (ISD::AssertSext).
199  const SDValue *Parts, unsigned NumParts,
200  MVT PartVT, EVT ValueVT, const Value *V,
201  Optional<CallingConv::ID> CC = None,
202  Optional<ISD::NodeType> AssertOp = None) {
203  if (ValueVT.isVector())
204  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
205  CC);
206 
207  assert(NumParts > 0 && "No parts to assemble!");
208  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
209  SDValue Val = Parts[0];
210 
211  if (NumParts > 1) {
212  // Assemble the value from multiple parts.
213  if (ValueVT.isInteger()) {
214  unsigned PartBits = PartVT.getSizeInBits();
215  unsigned ValueBits = ValueVT.getSizeInBits();
216 
217  // Assemble the power of 2 part.
218  unsigned RoundParts =
219  (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
220  unsigned RoundBits = PartBits * RoundParts;
221  EVT RoundVT = RoundBits == ValueBits ?
222  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
223  SDValue Lo, Hi;
224 
225  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
226 
227  if (RoundParts > 2) {
228  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
229  PartVT, HalfVT, V);
230  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
231  RoundParts / 2, PartVT, HalfVT, V);
232  } else {
233  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
234  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
235  }
236 
237  if (DAG.getDataLayout().isBigEndian())
238  std::swap(Lo, Hi);
239 
240  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
241 
242  if (RoundParts < NumParts) {
243  // Assemble the trailing non-power-of-2 part.
244  unsigned OddParts = NumParts - RoundParts;
245  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
246  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
247  OddVT, V, CC);
248 
249  // Combine the round and odd parts.
250  Lo = Val;
251  if (DAG.getDataLayout().isBigEndian())
252  std::swap(Lo, Hi);
253  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
254  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
255  Hi =
256  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
257  DAG.getConstant(Lo.getValueSizeInBits(), DL,
258  TLI.getPointerTy(DAG.getDataLayout())));
259  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
260  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
261  }
262  } else if (PartVT.isFloatingPoint()) {
263  // FP split into multiple FP parts (for ppcf128)
264  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
265  "Unexpected split");
266  SDValue Lo, Hi;
267  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
268  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
269  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
270  std::swap(Lo, Hi);
271  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
272  } else {
273  // FP split into integer parts (soft fp)
274  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
275  !PartVT.isVector() && "Unexpected split");
276  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
277  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
278  }
279  }
280 
281  // There is now one part, held in Val. Correct it to match ValueVT.
282  // PartEVT is the type of the register class that holds the value.
283  // ValueVT is the type of the inline asm operation.
284  EVT PartEVT = Val.getValueType();
285 
286  if (PartEVT == ValueVT)
287  return Val;
288 
289  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
290  ValueVT.bitsLT(PartEVT)) {
291  // For an FP value in an integer part, we need to truncate to the right
292  // width first.
293  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
294  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
295  }
296 
297  // Handle types that have the same size.
298  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
299  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
300 
301  // Handle types with different sizes.
302  if (PartEVT.isInteger() && ValueVT.isInteger()) {
303  if (ValueVT.bitsLT(PartEVT)) {
304  // For a truncate, see if we have any information to
305  // indicate whether the truncated bits will always be
306  // zero or sign-extension.
307  if (AssertOp.hasValue())
308  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
309  DAG.getValueType(ValueVT));
310  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
311  }
312  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
313  }
314 
315  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
316  // FP_ROUND's are always exact here.
317  if (ValueVT.bitsLT(Val.getValueType()))
318  return DAG.getNode(
319  ISD::FP_ROUND, DL, ValueVT, Val,
320  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
321 
322  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
323  }
324 
325  // Handle MMX to a narrower integer type by bitcasting MMX to integer and
326  // then truncating.
327  if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
328  ValueVT.bitsLT(PartEVT)) {
329  Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
330  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
331  }
332 
333  report_fatal_error("Unknown mismatch in getCopyFromParts!");
334 }
335 
337  const Twine &ErrMsg) {
338  const Instruction *I = dyn_cast_or_null<Instruction>(V);
339  if (!V)
340  return Ctx.emitError(ErrMsg);
341 
342  const char *AsmError = ", possible invalid constraint for vector type";
343  if (const CallInst *CI = dyn_cast<CallInst>(I))
344  if (isa<InlineAsm>(CI->getCalledValue()))
345  return Ctx.emitError(I, ErrMsg + AsmError);
346 
347  return Ctx.emitError(I, ErrMsg);
348 }
349 
350 /// getCopyFromPartsVector - Create a value that contains the specified legal
351 /// parts combined into the value they represent. If the parts combine to a
352 /// type larger than ValueVT then AssertOp can be used to specify whether the
353 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
354 /// ValueVT (ISD::AssertSext).
356  const SDValue *Parts, unsigned NumParts,
357  MVT PartVT, EVT ValueVT, const Value *V,
358  Optional<CallingConv::ID> CallConv) {
359  assert(ValueVT.isVector() && "Not a vector value");
360  assert(NumParts > 0 && "No parts to assemble!");
361  const bool IsABIRegCopy = CallConv.hasValue();
362 
363  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
364  SDValue Val = Parts[0];
365 
366  // Handle a multi-element vector.
367  if (NumParts > 1) {
368  EVT IntermediateVT;
369  MVT RegisterVT;
370  unsigned NumIntermediates;
371  unsigned NumRegs;
372 
373  if (IsABIRegCopy) {
375  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
376  NumIntermediates, RegisterVT);
377  } else {
378  NumRegs =
379  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
380  NumIntermediates, RegisterVT);
381  }
382 
383  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
384  NumParts = NumRegs; // Silence a compiler warning.
385  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
386  assert(RegisterVT.getSizeInBits() ==
387  Parts[0].getSimpleValueType().getSizeInBits() &&
388  "Part type sizes don't match!");
389 
390  // Assemble the parts into intermediate operands.
391  SmallVector<SDValue, 8> Ops(NumIntermediates);
392  if (NumIntermediates == NumParts) {
393  // If the register was not expanded, truncate or copy the value,
394  // as appropriate.
395  for (unsigned i = 0; i != NumParts; ++i)
396  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
397  PartVT, IntermediateVT, V);
398  } else if (NumParts > 0) {
399  // If the intermediate type was expanded, build the intermediate
400  // operands from the parts.
401  assert(NumParts % NumIntermediates == 0 &&
402  "Must expand into a divisible number of parts!");
403  unsigned Factor = NumParts / NumIntermediates;
404  for (unsigned i = 0; i != NumIntermediates; ++i)
405  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
406  PartVT, IntermediateVT, V);
407  }
408 
409  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
410  // intermediate operands.
411  EVT BuiltVectorTy =
412  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
413  (IntermediateVT.isVector()
414  ? IntermediateVT.getVectorNumElements() * NumParts
415  : NumIntermediates));
416  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
418  DL, BuiltVectorTy, Ops);
419  }
420 
421  // There is now one part, held in Val. Correct it to match ValueVT.
422  EVT PartEVT = Val.getValueType();
423 
424  if (PartEVT == ValueVT)
425  return Val;
426 
427  if (PartEVT.isVector()) {
428  // If the element type of the source/dest vectors are the same, but the
429  // parts vector has more elements than the value vector, then we have a
430  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
431  // elements we want.
432  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
433  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
434  "Cannot narrow, it would be a lossy transformation");
435  return DAG.getNode(
436  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
437  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
438  }
439 
440  // Vector/Vector bitcast.
441  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
442  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
443 
444  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
445  "Cannot handle this kind of promotion");
446  // Promoted vector extract
447  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
448 
449  }
450 
451  // Trivial bitcast if the types are the same size and the destination
452  // vector type is legal.
453  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
454  TLI.isTypeLegal(ValueVT))
455  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
456 
457  if (ValueVT.getVectorNumElements() != 1) {
458  // Certain ABIs require that vectors are passed as integers. For vectors
459  // are the same size, this is an obvious bitcast.
460  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
461  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
462  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
463  // Bitcast Val back the original type and extract the corresponding
464  // vector we want.
465  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
466  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
467  ValueVT.getVectorElementType(), Elts);
468  Val = DAG.getBitcast(WiderVecType, Val);
469  return DAG.getNode(
470  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
471  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
472  }
473 
475  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
476  return DAG.getUNDEF(ValueVT);
477  }
478 
479  // Handle cases such as i8 -> <1 x i1>
480  EVT ValueSVT = ValueVT.getVectorElementType();
481  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
482  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
483  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
484 
485  return DAG.getBuildVector(ValueVT, DL, Val);
486 }
487 
488 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
489  SDValue Val, SDValue *Parts, unsigned NumParts,
490  MVT PartVT, const Value *V,
491  Optional<CallingConv::ID> CallConv);
492 
493 /// getCopyToParts - Create a series of nodes that contain the specified value
494 /// split into legal parts. If the parts contain more bits than Val, then, for
495 /// integers, ExtendKind can be used to specify how to generate the extra bits.
496 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
497  SDValue *Parts, unsigned NumParts, MVT PartVT,
498  const Value *V,
499  Optional<CallingConv::ID> CallConv = None,
500  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
501  EVT ValueVT = Val.getValueType();
502 
503  // Handle the vector case separately.
504  if (ValueVT.isVector())
505  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
506  CallConv);
507 
508  unsigned PartBits = PartVT.getSizeInBits();
509  unsigned OrigNumParts = NumParts;
510  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
511  "Copying to an illegal type!");
512 
513  if (NumParts == 0)
514  return;
515 
516  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
517  EVT PartEVT = PartVT;
518  if (PartEVT == ValueVT) {
519  assert(NumParts == 1 && "No-op copy with multiple parts!");
520  Parts[0] = Val;
521  return;
522  }
523 
524  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
525  // If the parts cover more bits than the value has, promote the value.
526  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
527  assert(NumParts == 1 && "Do not know what to promote to!");
528  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
529  } else {
530  if (ValueVT.isFloatingPoint()) {
531  // FP values need to be bitcast, then extended if they are being put
532  // into a larger container.
533  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
534  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
535  }
536  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
537  ValueVT.isInteger() &&
538  "Unknown mismatch!");
539  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
540  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
541  if (PartVT == MVT::x86mmx)
542  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
543  }
544  } else if (PartBits == ValueVT.getSizeInBits()) {
545  // Different types of the same size.
546  assert(NumParts == 1 && PartEVT != ValueVT);
547  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
548  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
549  // If the parts cover less bits than value has, truncate the value.
550  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
551  ValueVT.isInteger() &&
552  "Unknown mismatch!");
553  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
554  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
555  if (PartVT == MVT::x86mmx)
556  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
557  }
558 
559  // The value may have changed - recompute ValueVT.
560  ValueVT = Val.getValueType();
561  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
562  "Failed to tile the value with PartVT!");
563 
564  if (NumParts == 1) {
565  if (PartEVT != ValueVT) {
567  "scalar-to-vector conversion failed");
568  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
569  }
570 
571  Parts[0] = Val;
572  return;
573  }
574 
575  // Expand the value into multiple parts.
576  if (NumParts & (NumParts - 1)) {
577  // The number of parts is not a power of 2. Split off and copy the tail.
578  assert(PartVT.isInteger() && ValueVT.isInteger() &&
579  "Do not know what to expand to!");
580  unsigned RoundParts = 1 << Log2_32(NumParts);
581  unsigned RoundBits = RoundParts * PartBits;
582  unsigned OddParts = NumParts - RoundParts;
583  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
584  DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
585 
586  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
587  CallConv);
588 
589  if (DAG.getDataLayout().isBigEndian())
590  // The odd parts were reversed by getCopyToParts - unreverse them.
591  std::reverse(Parts + RoundParts, Parts + NumParts);
592 
593  NumParts = RoundParts;
594  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
595  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
596  }
597 
598  // The number of parts is a power of 2. Repeatedly bisect the value using
599  // EXTRACT_ELEMENT.
600  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
602  ValueVT.getSizeInBits()),
603  Val);
604 
605  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
606  for (unsigned i = 0; i < NumParts; i += StepSize) {
607  unsigned ThisBits = StepSize * PartBits / 2;
608  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
609  SDValue &Part0 = Parts[i];
610  SDValue &Part1 = Parts[i+StepSize/2];
611 
612  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
613  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
614  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
615  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
616 
617  if (ThisBits == PartBits && ThisVT != PartVT) {
618  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
619  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
620  }
621  }
622  }
623 
624  if (DAG.getDataLayout().isBigEndian())
625  std::reverse(Parts, Parts + OrigNumParts);
626 }
627 
629  SDValue Val, const SDLoc &DL, EVT PartVT) {
630  if (!PartVT.isVector())
631  return SDValue();
632 
633  EVT ValueVT = Val.getValueType();
634  unsigned PartNumElts = PartVT.getVectorNumElements();
635  unsigned ValueNumElts = ValueVT.getVectorNumElements();
636  if (PartNumElts > ValueNumElts &&
637  PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
638  EVT ElementVT = PartVT.getVectorElementType();
639  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
640  // undef elements.
642  DAG.ExtractVectorElements(Val, Ops);
643  SDValue EltUndef = DAG.getUNDEF(ElementVT);
644  for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
645  Ops.push_back(EltUndef);
646 
647  // FIXME: Use CONCAT for 2x -> 4x.
648  return DAG.getBuildVector(PartVT, DL, Ops);
649  }
650 
651  return SDValue();
652 }
653 
654 /// getCopyToPartsVector - Create a series of nodes that contain the specified
655 /// value split into legal parts.
656 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
657  SDValue Val, SDValue *Parts, unsigned NumParts,
658  MVT PartVT, const Value *V,
659  Optional<CallingConv::ID> CallConv) {
660  EVT ValueVT = Val.getValueType();
661  assert(ValueVT.isVector() && "Not a vector");
662  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
663  const bool IsABIRegCopy = CallConv.hasValue();
664 
665  if (NumParts == 1) {
666  EVT PartEVT = PartVT;
667  if (PartEVT == ValueVT) {
668  // Nothing to do.
669  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
670  // Bitconvert vector->vector case.
671  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
672  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
673  Val = Widened;
674  } else if (PartVT.isVector() &&
675  PartEVT.getVectorElementType().bitsGE(
676  ValueVT.getVectorElementType()) &&
677  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
678 
679  // Promoted vector extract
680  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
681  } else {
682  if (ValueVT.getVectorNumElements() == 1) {
683  Val = DAG.getNode(
684  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
685  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
686  } else {
687  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
688  "lossy conversion of vector to scalar type");
689  EVT IntermediateType =
690  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
691  Val = DAG.getBitcast(IntermediateType, Val);
692  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
693  }
694  }
695 
696  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
697  Parts[0] = Val;
698  return;
699  }
700 
701  // Handle a multi-element vector.
702  EVT IntermediateVT;
703  MVT RegisterVT;
704  unsigned NumIntermediates;
705  unsigned NumRegs;
706  if (IsABIRegCopy) {
707  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
708  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
709  NumIntermediates, RegisterVT);
710  } else {
711  NumRegs =
712  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
713  NumIntermediates, RegisterVT);
714  }
715 
716  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
717  NumParts = NumRegs; // Silence a compiler warning.
718  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
719 
720  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
721  IntermediateVT.getVectorNumElements() : 1;
722 
723  // Convert the vector to the appropiate type if necessary.
724  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
725 
726  EVT BuiltVectorTy = EVT::getVectorVT(
727  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
728  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
729  if (ValueVT != BuiltVectorTy) {
730  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
731  Val = Widened;
732 
733  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
734  }
735 
736  // Split the vector into intermediate operands.
737  SmallVector<SDValue, 8> Ops(NumIntermediates);
738  for (unsigned i = 0; i != NumIntermediates; ++i) {
739  if (IntermediateVT.isVector()) {
740  Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
741  DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
742  } else {
743  Ops[i] = DAG.getNode(
744  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
745  DAG.getConstant(i, DL, IdxVT));
746  }
747  }
748 
749  // Split the intermediate operands into legal parts.
750  if (NumParts == NumIntermediates) {
751  // If the register was not expanded, promote or copy the value,
752  // as appropriate.
753  for (unsigned i = 0; i != NumParts; ++i)
754  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
755  } else if (NumParts > 0) {
756  // If the intermediate type was expanded, split each the value into
757  // legal parts.
758  assert(NumIntermediates != 0 && "division by zero");
759  assert(NumParts % NumIntermediates == 0 &&
760  "Must expand into a divisible number of parts!");
761  unsigned Factor = NumParts / NumIntermediates;
762  for (unsigned i = 0; i != NumIntermediates; ++i)
763  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
764  CallConv);
765  }
766 }
767 
769  EVT valuevt, Optional<CallingConv::ID> CC)
770  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
771  RegCount(1, regs.size()), CallConv(CC) {}
772 
774  const DataLayout &DL, unsigned Reg, Type *Ty,
776  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
777 
778  CallConv = CC;
779 
780  for (EVT ValueVT : ValueVTs) {
781  unsigned NumRegs =
782  isABIMangled()
783  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
784  : TLI.getNumRegisters(Context, ValueVT);
785  MVT RegisterVT =
786  isABIMangled()
787  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
788  : TLI.getRegisterType(Context, ValueVT);
789  for (unsigned i = 0; i != NumRegs; ++i)
790  Regs.push_back(Reg + i);
791  RegVTs.push_back(RegisterVT);
792  RegCount.push_back(NumRegs);
793  Reg += NumRegs;
794  }
795 }
796 
798  FunctionLoweringInfo &FuncInfo,
799  const SDLoc &dl, SDValue &Chain,
800  SDValue *Flag, const Value *V) const {
801  // A Value with type {} or [0 x %t] needs no registers.
802  if (ValueVTs.empty())
803  return SDValue();
804 
805  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
806 
807  // Assemble the legal parts into the final values.
808  SmallVector<SDValue, 4> Values(ValueVTs.size());
810  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
811  // Copy the legal parts from the registers.
812  EVT ValueVT = ValueVTs[Value];
813  unsigned NumRegs = RegCount[Value];
814  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
815  *DAG.getContext(),
816  CallConv.getValue(), RegVTs[Value])
817  : RegVTs[Value];
818 
819  Parts.resize(NumRegs);
820  for (unsigned i = 0; i != NumRegs; ++i) {
821  SDValue P;
822  if (!Flag) {
823  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
824  } else {
825  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
826  *Flag = P.getValue(2);
827  }
828 
829  Chain = P.getValue(1);
830  Parts[i] = P;
831 
832  // If the source register was virtual and if we know something about it,
833  // add an assert node.
835  !RegisterVT.isInteger())
836  continue;
837 
839  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
840  if (!LOI)
841  continue;
842 
843  unsigned RegSize = RegisterVT.getScalarSizeInBits();
844  unsigned NumSignBits = LOI->NumSignBits;
845  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
846 
847  if (NumZeroBits == RegSize) {
848  // The current value is a zero.
849  // Explicitly express that as it would be easier for
850  // optimizations to kick in.
851  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
852  continue;
853  }
854 
855  // FIXME: We capture more information than the dag can represent. For
856  // now, just use the tightest assertzext/assertsext possible.
857  bool isSExt;
858  EVT FromVT(MVT::Other);
859  if (NumZeroBits) {
860  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
861  isSExt = false;
862  } else if (NumSignBits > 1) {
863  FromVT =
864  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
865  isSExt = true;
866  } else {
867  continue;
868  }
869  // Add an assertion node.
870  assert(FromVT != MVT::Other);
871  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
872  RegisterVT, P, DAG.getValueType(FromVT));
873  }
874 
875  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
876  RegisterVT, ValueVT, V, CallConv);
877  Part += NumRegs;
878  Parts.clear();
879  }
880 
881  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
882 }
883 
885  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
886  const Value *V,
887  ISD::NodeType PreferredExtendType) const {
888  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
889  ISD::NodeType ExtendKind = PreferredExtendType;
890 
891  // Get the list of the values's legal parts.
892  unsigned NumRegs = Regs.size();
893  SmallVector<SDValue, 8> Parts(NumRegs);
894  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
895  unsigned NumParts = RegCount[Value];
896 
897  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
898  *DAG.getContext(),
899  CallConv.getValue(), RegVTs[Value])
900  : RegVTs[Value];
901 
902  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
903  ExtendKind = ISD::ZERO_EXTEND;
904 
905  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
906  NumParts, RegisterVT, V, CallConv, ExtendKind);
907  Part += NumParts;
908  }
909 
910  // Copy the parts into the registers.
911  SmallVector<SDValue, 8> Chains(NumRegs);
912  for (unsigned i = 0; i != NumRegs; ++i) {
913  SDValue Part;
914  if (!Flag) {
915  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
916  } else {
917  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
918  *Flag = Part.getValue(1);
919  }
920 
921  Chains[i] = Part.getValue(0);
922  }
923 
924  if (NumRegs == 1 || Flag)
925  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
926  // flagged to it. That is the CopyToReg nodes and the user are considered
927  // a single scheduling unit. If we create a TokenFactor and return it as
928  // chain, then the TokenFactor is both a predecessor (operand) of the
929  // user as well as a successor (the TF operands are flagged to the user).
930  // c1, f1 = CopyToReg
931  // c2, f2 = CopyToReg
932  // c3 = TokenFactor c1, c2
933  // ...
934  // = op c3, ..., f2
935  Chain = Chains[NumRegs-1];
936  else
937  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
938 }
939 
940 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
941  unsigned MatchingIdx, const SDLoc &dl,
942  SelectionDAG &DAG,
943  std::vector<SDValue> &Ops) const {
944  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
945 
946  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
947  if (HasMatching)
948  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
949  else if (!Regs.empty() &&
951  // Put the register class of the virtual registers in the flag word. That
952  // way, later passes can recompute register class constraints for inline
953  // assembly as well as normal instructions.
954  // Don't do this for tied operands that can use the regclass information
955  // from the def.
957  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
958  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
959  }
960 
961  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
962  Ops.push_back(Res);
963 
964  if (Code == InlineAsm::Kind_Clobber) {
965  // Clobbers should always have a 1:1 mapping with registers, and may
966  // reference registers that have illegal (e.g. vector) types. Hence, we
967  // shouldn't try to apply any sort of splitting logic to them.
968  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
969  "No 1:1 mapping from clobbers to regs?");
970  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
971  (void)SP;
972  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
973  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
974  assert(
975  (Regs[I] != SP ||
977  "If we clobbered the stack pointer, MFI should know about it.");
978  }
979  return;
980  }
981 
982  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
983  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
984  MVT RegisterVT = RegVTs[Value];
985  for (unsigned i = 0; i != NumRegs; ++i) {
986  assert(Reg < Regs.size() && "Mismatch in # registers expected");
987  unsigned TheReg = Regs[Reg++];
988  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
989  }
990  }
991 }
992 
996  unsigned I = 0;
997  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
998  unsigned RegCount = std::get<0>(CountAndVT);
999  MVT RegisterVT = std::get<1>(CountAndVT);
1000  unsigned RegisterSize = RegisterVT.getSizeInBits();
1001  for (unsigned E = I + RegCount; I != E; ++I)
1002  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1003  }
1004  return OutVec;
1005 }
1006 
1008  const TargetLibraryInfo *li) {
1009  AA = aa;
1010  GFI = gfi;
1011  LibInfo = li;
1012  DL = &DAG.getDataLayout();
1013  Context = DAG.getContext();
1014  LPadToCallSiteMap.clear();
1015 }
1016 
1018  NodeMap.clear();
1019  UnusedArgNodeMap.clear();
1020  PendingLoads.clear();
1021  PendingExports.clear();
1022  CurInst = nullptr;
1023  HasTailCall = false;
1024  SDNodeOrder = LowestSDNodeOrder;
1025  StatepointLowering.clear();
1026 }
1027 
1029  DanglingDebugInfoMap.clear();
1030 }
1031 
1033  if (PendingLoads.empty())
1034  return DAG.getRoot();
1035 
1036  if (PendingLoads.size() == 1) {
1037  SDValue Root = PendingLoads[0];
1038  DAG.setRoot(Root);
1039  PendingLoads.clear();
1040  return Root;
1041  }
1042 
1043  // Otherwise, we have to make a token factor node.
1044  SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1045  PendingLoads.clear();
1046  DAG.setRoot(Root);
1047  return Root;
1048 }
1049 
1051  SDValue Root = DAG.getRoot();
1052 
1053  if (PendingExports.empty())
1054  return Root;
1055 
1056  // Turn all of the CopyToReg chains into one factored node.
1057  if (Root.getOpcode() != ISD::EntryToken) {
1058  unsigned i = 0, e = PendingExports.size();
1059  for (; i != e; ++i) {
1060  assert(PendingExports[i].getNode()->getNumOperands() > 1);
1061  if (PendingExports[i].getNode()->getOperand(0) == Root)
1062  break; // Don't add the root if we already indirectly depend on it.
1063  }
1064 
1065  if (i == e)
1066  PendingExports.push_back(Root);
1067  }
1068 
1069  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1070  PendingExports);
1071  PendingExports.clear();
1072  DAG.setRoot(Root);
1073  return Root;
1074 }
1075 
1077  // Set up outgoing PHI node register values before emitting the terminator.
1078  if (I.isTerminator()) {
1079  HandlePHINodesInSuccessorBlocks(I.getParent());
1080  }
1081 
1082  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1083  if (!isa<DbgInfoIntrinsic>(I))
1084  ++SDNodeOrder;
1085 
1086  CurInst = &I;
1087 
1088  visit(I.getOpcode(), I);
1089 
1090  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1091  // Propagate the fast-math-flags of this IR instruction to the DAG node that
1092  // maps to this instruction.
1093  // TODO: We could handle all flags (nsw, etc) here.
1094  // TODO: If an IR instruction maps to >1 node, only the final node will have
1095  // flags set.
1096  if (SDNode *Node = getNodeForIRValue(&I)) {
1097  SDNodeFlags IncomingFlags;
1098  IncomingFlags.copyFMF(*FPMO);
1099  if (!Node->getFlags().isDefined())
1100  Node->setFlags(IncomingFlags);
1101  else
1102  Node->intersectFlagsWith(IncomingFlags);
1103  }
1104  }
1105 
1106  if (!I.isTerminator() && !HasTailCall &&
1107  !isStatepoint(&I)) // statepoints handle their exports internally
1108  CopyToExportRegsIfNeeded(&I);
1109 
1110  CurInst = nullptr;
1111 }
1112 
1113 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1114  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1115 }
1116 
1117 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1118  // Note: this doesn't use InstVisitor, because it has to work with
1119  // ConstantExpr's in addition to instructions.
1120  switch (Opcode) {
1121  default: llvm_unreachable("Unknown instruction type encountered!");
1122  // Build the switch statement using the Instruction.def file.
1123 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1124  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1125 #include "llvm/IR/Instruction.def"
1126  }
1127 }
1128 
1130  const DIExpression *Expr) {
1131  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1132  const DbgValueInst *DI = DDI.getDI();
1133  DIVariable *DanglingVariable = DI->getVariable();
1134  DIExpression *DanglingExpr = DI->getExpression();
1135  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1136  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1137  return true;
1138  }
1139  return false;
1140  };
1141 
1142  for (auto &DDIMI : DanglingDebugInfoMap) {
1143  DanglingDebugInfoVector &DDIV = DDIMI.second;
1144 
1145  // If debug info is to be dropped, run it through final checks to see
1146  // whether it can be salvaged.
1147  for (auto &DDI : DDIV)
1148  if (isMatchingDbgValue(DDI))
1149  salvageUnresolvedDbgValue(DDI);
1150 
1151  DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1152  }
1153 }
1154 
1155 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1156 // generate the debug data structures now that we've seen its definition.
1158  SDValue Val) {
1159  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1160  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1161  return;
1162 
1163  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1164  for (auto &DDI : DDIV) {
1165  const DbgValueInst *DI = DDI.getDI();
1166  assert(DI && "Ill-formed DanglingDebugInfo");
1167  DebugLoc dl = DDI.getdl();
1168  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1169  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1170  DILocalVariable *Variable = DI->getVariable();
1171  DIExpression *Expr = DI->getExpression();
1172  assert(Variable->isValidLocationForIntrinsic(dl) &&
1173  "Expected inlined-at fields to agree");
1174  SDDbgValue *SDV;
1175  if (Val.getNode()) {
1176  // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1177  // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1178  // we couldn't resolve it directly when examining the DbgValue intrinsic
1179  // in the first place we should not be more successful here). Unless we
1180  // have some test case that prove this to be correct we should avoid
1181  // calling EmitFuncArgumentDbgValue here.
1182  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1183  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1184  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1185  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1186  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1187  // inserted after the definition of Val when emitting the instructions
1188  // after ISel. An alternative could be to teach
1189  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1190  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1191  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1192  << ValSDNodeOrder << "\n");
1193  SDV = getDbgValue(Val, Variable, Expr, dl,
1194  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1195  DAG.AddDbgValue(SDV, Val.getNode(), false);
1196  } else
1197  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1198  << "in EmitFuncArgumentDbgValue\n");
1199  } else {
1200  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1201  auto Undef =
1202  UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1203  auto SDV =
1204  DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1205  DAG.AddDbgValue(SDV, nullptr, false);
1206  }
1207  }
1208  DDIV.clear();
1209 }
1210 
1212  Value *V = DDI.getDI()->getValue();
1213  DILocalVariable *Var = DDI.getDI()->getVariable();
1214  DIExpression *Expr = DDI.getDI()->getExpression();
1215  DebugLoc DL = DDI.getdl();
1216  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1217  unsigned SDOrder = DDI.getSDNodeOrder();
1218 
1219  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1220  // that DW_OP_stack_value is desired.
1221  assert(isa<DbgValueInst>(DDI.getDI()));
1222  bool StackValue = true;
1223 
1224  // Can this Value can be encoded without any further work?
1225  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1226  return;
1227 
1228  // Attempt to salvage back through as many instructions as possible. Bail if
1229  // a non-instruction is seen, such as a constant expression or global
1230  // variable. FIXME: Further work could recover those too.
1231  while (isa<Instruction>(V)) {
1232  Instruction &VAsInst = *cast<Instruction>(V);
1233  DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
1234 
1235  // If we cannot salvage any further, and haven't yet found a suitable debug
1236  // expression, bail out.
1237  if (!NewExpr)
1238  break;
1239 
1240  // New value and expr now represent this debuginfo.
1241  V = VAsInst.getOperand(0);
1242  Expr = NewExpr;
1243 
1244  // Some kind of simplification occurred: check whether the operand of the
1245  // salvaged debug expression can be encoded in this DAG.
1246  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1247  LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "
1248  << DDI.getDI() << "\nBy stripping back to:\n " << V);
1249  return;
1250  }
1251  }
1252 
1253  // This was the final opportunity to salvage this debug information, and it
1254  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1255  // any earlier variable location.
1256  auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1257  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1258  DAG.AddDbgValue(SDV, nullptr, false);
1259 
1260  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()
1261  << "\n");
1262  LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)
1263  << "\n");
1264 }
1265 
1267  DIExpression *Expr, DebugLoc dl,
1268  DebugLoc InstDL, unsigned Order) {
1269  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1270  SDDbgValue *SDV;
1271  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1272  isa<ConstantPointerNull>(V)) {
1273  SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
1274  DAG.AddDbgValue(SDV, nullptr, false);
1275  return true;
1276  }
1277 
1278  // If the Value is a frame index, we can create a FrameIndex debug value
1279  // without relying on the DAG at all.
1280  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1281  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1282  if (SI != FuncInfo.StaticAllocaMap.end()) {
1283  auto SDV =
1284  DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
1285  /*IsIndirect*/ false, dl, SDNodeOrder);
1286  // Do not attach the SDNodeDbgValue to an SDNode: this variable location
1287  // is still available even if the SDNode gets optimized out.
1288  DAG.AddDbgValue(SDV, nullptr, false);
1289  return true;
1290  }
1291  }
1292 
1293  // Do not use getValue() in here; we don't want to generate code at
1294  // this point if it hasn't been done yet.
1295  SDValue N = NodeMap[V];
1296  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1297  N = UnusedArgNodeMap[V];
1298  if (N.getNode()) {
1299  if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1300  return true;
1301  SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1302  DAG.AddDbgValue(SDV, N.getNode(), false);
1303  return true;
1304  }
1305 
1306  // Special rules apply for the first dbg.values of parameter variables in a
1307  // function. Identify them by the fact they reference Argument Values, that
1308  // they're parameters, and they are parameters of the current function. We
1309  // need to let them dangle until they get an SDNode.
1310  bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&
1311  !InstDL.getInlinedAt();
1312  if (!IsParamOfFunc) {
1313  // The value is not used in this block yet (or it would have an SDNode).
1314  // We still want the value to appear for the user if possible -- if it has
1315  // an associated VReg, we can refer to that instead.
1316  auto VMI = FuncInfo.ValueMap.find(V);
1317  if (VMI != FuncInfo.ValueMap.end()) {
1318  unsigned Reg = VMI->second;
1319  // If this is a PHI node, it may be split up into several MI PHI nodes
1320  // (in FunctionLoweringInfo::set).
1321  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1322  V->getType(), None);
1323  if (RFV.occupiesMultipleRegs()) {
1324  unsigned Offset = 0;
1325  unsigned BitsToDescribe = 0;
1326  if (auto VarSize = Var->getSizeInBits())
1327  BitsToDescribe = *VarSize;
1328  if (auto Fragment = Expr->getFragmentInfo())
1329  BitsToDescribe = Fragment->SizeInBits;
1330  for (auto RegAndSize : RFV.getRegsAndSizes()) {
1331  unsigned RegisterSize = RegAndSize.second;
1332  // Bail out if all bits are described already.
1333  if (Offset >= BitsToDescribe)
1334  break;
1335  unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1336  ? BitsToDescribe - Offset
1337  : RegisterSize;
1338  auto FragmentExpr = DIExpression::createFragmentExpression(
1339  Expr, Offset, FragmentSize);
1340  if (!FragmentExpr)
1341  continue;
1342  SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
1343  false, dl, SDNodeOrder);
1344  DAG.AddDbgValue(SDV, nullptr, false);
1345  Offset += RegisterSize;
1346  }
1347  } else {
1348  SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
1349  DAG.AddDbgValue(SDV, nullptr, false);
1350  }
1351  return true;
1352  }
1353  }
1354 
1355  return false;
1356 }
1357 
1359  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1360  for (auto &Pair : DanglingDebugInfoMap)
1361  for (auto &DDI : Pair.second)
1362  salvageUnresolvedDbgValue(DDI);
1363  clearDanglingDebugInfo();
1364 }
1365 
1366 /// getCopyFromRegs - If there was virtual register allocated for the value V
1367 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1369  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1370  SDValue Result;
1371 
1372  if (It != FuncInfo.ValueMap.end()) {
1373  unsigned InReg = It->second;
1374 
1375  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1376  DAG.getDataLayout(), InReg, Ty,
1377  None); // This is not an ABI copy.
1378  SDValue Chain = DAG.getEntryNode();
1379  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1380  V);
1381  resolveDanglingDebugInfo(V, Result);
1382  }
1383 
1384  return Result;
1385 }
1386 
1387 /// getValue - Return an SDValue for the given Value.
1389  // If we already have an SDValue for this value, use it. It's important
1390  // to do this first, so that we don't create a CopyFromReg if we already
1391  // have a regular SDValue.
1392  SDValue &N = NodeMap[V];
1393  if (N.getNode()) return N;
1394 
1395  // If there's a virtual register allocated and initialized for this
1396  // value, use it.
1397  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1398  return copyFromReg;
1399 
1400  // Otherwise create a new SDValue and remember it.
1401  SDValue Val = getValueImpl(V);
1402  NodeMap[V] = Val;
1403  resolveDanglingDebugInfo(V, Val);
1404  return Val;
1405 }
1406 
1407 // Return true if SDValue exists for the given Value
1409  return (NodeMap.find(V) != NodeMap.end()) ||
1410  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1411 }
1412 
1413 /// getNonRegisterValue - Return an SDValue for the given Value, but
1414 /// don't look in FuncInfo.ValueMap for a virtual register.
1416  // If we already have an SDValue for this value, use it.
1417  SDValue &N = NodeMap[V];
1418  if (N.getNode()) {
1419  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1420  // Remove the debug location from the node as the node is about to be used
1421  // in a location which may differ from the original debug location. This
1422  // is relevant to Constant and ConstantFP nodes because they can appear
1423  // as constant expressions inside PHI nodes.
1424  N->setDebugLoc(DebugLoc());
1425  }
1426  return N;
1427  }
1428 
1429  // Otherwise create a new SDValue and remember it.
1430  SDValue Val = getValueImpl(V);
1431  NodeMap[V] = Val;
1432  resolveDanglingDebugInfo(V, Val);
1433  return Val;
1434 }
1435 
1436 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1437 /// Create an SDValue for the given value.
1439  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1440 
1441  if (const Constant *C = dyn_cast<Constant>(V)) {
1442  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1443 
1444  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1445  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1446 
1447  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1448  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1449 
1450  if (isa<ConstantPointerNull>(C)) {
1451  unsigned AS = V->getType()->getPointerAddressSpace();
1452  return DAG.getConstant(0, getCurSDLoc(),
1453  TLI.getPointerTy(DAG.getDataLayout(), AS));
1454  }
1455 
1456  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1457  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1458 
1459  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1460  return DAG.getUNDEF(VT);
1461 
1462  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1463  visit(CE->getOpcode(), *CE);
1464  SDValue N1 = NodeMap[V];
1465  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1466  return N1;
1467  }
1468 
1469  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1471  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1472  OI != OE; ++OI) {
1473  SDNode *Val = getValue(*OI).getNode();
1474  // If the operand is an empty aggregate, there are no values.
1475  if (!Val) continue;
1476  // Add each leaf value from the operand to the Constants list
1477  // to form a flattened list of all the values.
1478  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1479  Constants.push_back(SDValue(Val, i));
1480  }
1481 
1482  return DAG.getMergeValues(Constants, getCurSDLoc());
1483  }
1484 
1485  if (const ConstantDataSequential *CDS =
1486  dyn_cast<ConstantDataSequential>(C)) {
1488  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1489  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1490  // Add each leaf value from the operand to the Constants list
1491  // to form a flattened list of all the values.
1492  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1493  Ops.push_back(SDValue(Val, i));
1494  }
1495 
1496  if (isa<ArrayType>(CDS->getType()))
1497  return DAG.getMergeValues(Ops, getCurSDLoc());
1498  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1499  }
1500 
1501  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1502  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1503  "Unknown struct or array constant!");
1504 
1506  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1507  unsigned NumElts = ValueVTs.size();
1508  if (NumElts == 0)
1509  return SDValue(); // empty struct
1511  for (unsigned i = 0; i != NumElts; ++i) {
1512  EVT EltVT = ValueVTs[i];
1513  if (isa<UndefValue>(C))
1514  Constants[i] = DAG.getUNDEF(EltVT);
1515  else if (EltVT.isFloatingPoint())
1516  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1517  else
1518  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1519  }
1520 
1521  return DAG.getMergeValues(Constants, getCurSDLoc());
1522  }
1523 
1524  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1525  return DAG.getBlockAddress(BA, VT);
1526 
1527  VectorType *VecTy = cast<VectorType>(V->getType());
1528  unsigned NumElements = VecTy->getNumElements();
1529 
1530  // Now that we know the number and type of the elements, get that number of
1531  // elements into the Ops array based on what kind of constant it is.
1533  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1534  for (unsigned i = 0; i != NumElements; ++i)
1535  Ops.push_back(getValue(CV->getOperand(i)));
1536  } else {
1537  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1538  EVT EltVT =
1539  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1540 
1541  SDValue Op;
1542  if (EltVT.isFloatingPoint())
1543  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1544  else
1545  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1546  Ops.assign(NumElements, Op);
1547  }
1548 
1549  // Create a BUILD_VECTOR node.
1550  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1551  }
1552 
1553  // If this is a static alloca, generate it as the frameindex instead of
1554  // computation.
1555  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1557  FuncInfo.StaticAllocaMap.find(AI);
1558  if (SI != FuncInfo.StaticAllocaMap.end())
1559  return DAG.getFrameIndex(SI->second,
1560  TLI.getFrameIndexTy(DAG.getDataLayout()));
1561  }
1562 
1563  // If this is an instruction which fast-isel has deferred, select it now.
1564  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1565  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1566 
1567  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1568  Inst->getType(), getABIRegCopyCC(V));
1569  SDValue Chain = DAG.getEntryNode();
1570  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1571  }
1572 
1573  llvm_unreachable("Can't get register for value!");
1574 }
1575 
1576 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1577  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1578  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1579  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1580  bool IsSEH = isAsynchronousEHPersonality(Pers);
1581  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1582  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1583  if (!IsSEH)
1584  CatchPadMBB->setIsEHScopeEntry();
1585  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1586  if (IsMSVCCXX || IsCoreCLR)
1587  CatchPadMBB->setIsEHFuncletEntry();
1588  // Wasm does not need catchpads anymore
1589  if (!IsWasmCXX)
1590  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1591  getControlRoot()));
1592 }
1593 
1594 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1595  // Update machine-CFG edge.
1596  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1597  FuncInfo.MBB->addSuccessor(TargetMBB);
1598 
1599  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1600  bool IsSEH = isAsynchronousEHPersonality(Pers);
1601  if (IsSEH) {
1602  // If this is not a fall-through branch or optimizations are switched off,
1603  // emit the branch.
1604  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1605  TM.getOptLevel() == CodeGenOpt::None)
1606  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1607  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1608  return;
1609  }
1610 
1611  // Figure out the funclet membership for the catchret's successor.
1612  // This will be used by the FuncletLayout pass to determine how to order the
1613  // BB's.
1614  // A 'catchret' returns to the outer scope's color.
1615  Value *ParentPad = I.getCatchSwitchParentPad();
1616  const BasicBlock *SuccessorColor;
1617  if (isa<ConstantTokenNone>(ParentPad))
1618  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1619  else
1620  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1621  assert(SuccessorColor && "No parent funclet for catchret!");
1622  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1623  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1624 
1625  // Create the terminator node.
1626  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1627  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1628  DAG.getBasicBlock(SuccessorColorMBB));
1629  DAG.setRoot(Ret);
1630 }
1631 
1632 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1633  // Don't emit any special code for the cleanuppad instruction. It just marks
1634  // the start of an EH scope/funclet.
1635  FuncInfo.MBB->setIsEHScopeEntry();
1636  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1637  if (Pers != EHPersonality::Wasm_CXX) {
1638  FuncInfo.MBB->setIsEHFuncletEntry();
1639  FuncInfo.MBB->setIsCleanupFuncletEntry();
1640  }
1641 }
1642 
1643 // For wasm, there's alwyas a single catch pad attached to a catchswitch, and
1644 // the control flow always stops at the single catch pad, as it does for a
1645 // cleanup pad. In case the exception caught is not of the types the catch pad
1646 // catches, it will be rethrown by a rethrow.
1648  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1649  BranchProbability Prob,
1650  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1651  &UnwindDests) {
1652  while (EHPadBB) {
1653  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1654  if (isa<CleanupPadInst>(Pad)) {
1655  // Stop on cleanup pads.
1656  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1657  UnwindDests.back().first->setIsEHScopeEntry();
1658  break;
1659  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1660  // Add the catchpad handlers to the possible destinations. We don't
1661  // continue to the unwind destination of the catchswitch for wasm.
1662  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1663  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1664  UnwindDests.back().first->setIsEHScopeEntry();
1665  }
1666  break;
1667  } else {
1668  continue;
1669  }
1670  }
1671 }
1672 
1673 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1674 /// many places it could ultimately go. In the IR, we have a single unwind
1675 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1676 /// This function skips over imaginary basic blocks that hold catchswitch
1677 /// instructions, and finds all the "real" machine
1678 /// basic block destinations. As those destinations may not be successors of
1679 /// EHPadBB, here we also calculate the edge probability to those destinations.
1680 /// The passed-in Prob is the edge probability to EHPadBB.
1682  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1683  BranchProbability Prob,
1684  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1685  &UnwindDests) {
1686  EHPersonality Personality =
1688  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1689  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1690  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1691  bool IsSEH = isAsynchronousEHPersonality(Personality);
1692 
1693  if (IsWasmCXX) {
1694  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1695  assert(UnwindDests.size() <= 1 &&
1696  "There should be at most one unwind destination for wasm");
1697  return;
1698  }
1699 
1700  while (EHPadBB) {
1701  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1702  BasicBlock *NewEHPadBB = nullptr;
1703  if (isa<LandingPadInst>(Pad)) {
1704  // Stop on landingpads. They are not funclets.
1705  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1706  break;
1707  } else if (isa<CleanupPadInst>(Pad)) {
1708  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1709  // personalities.
1710  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1711  UnwindDests.back().first->setIsEHScopeEntry();
1712  UnwindDests.back().first->setIsEHFuncletEntry();
1713  break;
1714  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1715  // Add the catchpad handlers to the possible destinations.
1716  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1717  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1718  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1719  if (IsMSVCCXX || IsCoreCLR)
1720  UnwindDests.back().first->setIsEHFuncletEntry();
1721  if (!IsSEH)
1722  UnwindDests.back().first->setIsEHScopeEntry();
1723  }
1724  NewEHPadBB = CatchSwitch->getUnwindDest();
1725  } else {
1726  continue;
1727  }
1728 
1729  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1730  if (BPI && NewEHPadBB)
1731  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1732  EHPadBB = NewEHPadBB;
1733  }
1734 }
1735 
1736 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1737  // Update successor info.
1739  auto UnwindDest = I.getUnwindDest();
1740  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1741  BranchProbability UnwindDestProb =
1742  (BPI && UnwindDest)
1743  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1745  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1746  for (auto &UnwindDest : UnwindDests) {
1747  UnwindDest.first->setIsEHPad();
1748  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1749  }
1750  FuncInfo.MBB->normalizeSuccProbs();
1751 
1752  // Create the terminator node.
1753  SDValue Ret =
1754  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1755  DAG.setRoot(Ret);
1756 }
1757 
1758 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1759  report_fatal_error("visitCatchSwitch not yet implemented!");
1760 }
1761 
1762 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1763  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1764  auto &DL = DAG.getDataLayout();
1765  SDValue Chain = getControlRoot();
1767  SmallVector<SDValue, 8> OutVals;
1768 
1769  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1770  // lower
1771  //
1772  // %val = call <ty> @llvm.experimental.deoptimize()
1773  // ret <ty> %val
1774  //
1775  // differently.
1777  LowerDeoptimizingReturn();
1778  return;
1779  }
1780 
1781  if (!FuncInfo.CanLowerReturn) {
1782  unsigned DemoteReg = FuncInfo.DemoteRegister;
1783  const Function *F = I.getParent()->getParent();
1784 
1785  // Emit a store of the return value through the virtual register.
1786  // Leave Outs empty so that LowerReturn won't try to load return
1787  // registers the usual way.
1788  SmallVector<EVT, 1> PtrValueVTs;
1789  ComputeValueVTs(TLI, DL,
1792  PtrValueVTs);
1793 
1794  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1795  DemoteReg, PtrValueVTs[0]);
1796  SDValue RetOp = getValue(I.getOperand(0));
1797 
1798  SmallVector<EVT, 4> ValueVTs, MemVTs;
1800  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1801  &Offsets);
1802  unsigned NumValues = ValueVTs.size();
1803 
1804  SmallVector<SDValue, 4> Chains(NumValues);
1805  for (unsigned i = 0; i != NumValues; ++i) {
1806  // An aggregate return value cannot wrap around the address space, so
1807  // offsets to its parts don't wrap either.
1808  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1809 
1810  SDValue Val = RetOp.getValue(i);
1811  if (MemVTs[i] != ValueVTs[i])
1812  Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1813  Chains[i] = DAG.getStore(Chain, getCurSDLoc(), Val,
1814  // FIXME: better loc info would be nice.
1816  }
1817 
1818  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1819  MVT::Other, Chains);
1820  } else if (I.getNumOperands() != 0) {
1822  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1823  unsigned NumValues = ValueVTs.size();
1824  if (NumValues) {
1825  SDValue RetOp = getValue(I.getOperand(0));
1826 
1827  const Function *F = I.getParent()->getParent();
1828 
1829  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1830  I.getOperand(0)->getType(), F->getCallingConv(),
1831  /*IsVarArg*/ false);
1832 
1833  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1834  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1835  Attribute::SExt))
1836  ExtendKind = ISD::SIGN_EXTEND;
1837  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1838  Attribute::ZExt))
1839  ExtendKind = ISD::ZERO_EXTEND;
1840 
1841  LLVMContext &Context = F->getContext();
1842  bool RetInReg = F->getAttributes().hasAttribute(
1843  AttributeList::ReturnIndex, Attribute::InReg);
1844 
1845  for (unsigned j = 0; j != NumValues; ++j) {
1846  EVT VT = ValueVTs[j];
1847 
1848  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1849  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1850 
1851  CallingConv::ID CC = F->getCallingConv();
1852 
1853  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1854  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1855  SmallVector<SDValue, 4> Parts(NumParts);
1856  getCopyToParts(DAG, getCurSDLoc(),
1857  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1858  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1859 
1860  // 'inreg' on function refers to return value
1861  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1862  if (RetInReg)
1863  Flags.setInReg();
1864 
1865  if (I.getOperand(0)->getType()->isPointerTy()) {
1866  Flags.setPointer();
1867  Flags.setPointerAddrSpace(
1868  cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
1869  }
1870 
1871  if (NeedsRegBlock) {
1872  Flags.setInConsecutiveRegs();
1873  if (j == NumValues - 1)
1874  Flags.setInConsecutiveRegsLast();
1875  }
1876 
1877  // Propagate extension type if any
1878  if (ExtendKind == ISD::SIGN_EXTEND)
1879  Flags.setSExt();
1880  else if (ExtendKind == ISD::ZERO_EXTEND)
1881  Flags.setZExt();
1882 
1883  for (unsigned i = 0; i < NumParts; ++i) {
1884  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1885  VT, /*isfixed=*/true, 0, 0));
1886  OutVals.push_back(Parts[i]);
1887  }
1888  }
1889  }
1890  }
1891 
1892  // Push in swifterror virtual register as the last element of Outs. This makes
1893  // sure swifterror virtual register will be returned in the swifterror
1894  // physical register.
1895  const Function *F = I.getParent()->getParent();
1896  if (TLI.supportSwiftError() &&
1897  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1898  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1899  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1900  Flags.setSwiftError();
1901  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1902  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1903  true /*isfixed*/, 1 /*origidx*/,
1904  0 /*partOffs*/));
1905  // Create SDNode for the swifterror virtual register.
1906  OutVals.push_back(
1907  DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1908  &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1909  EVT(TLI.getPointerTy(DL))));
1910  }
1911 
1912  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1913  CallingConv::ID CallConv =
1915  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1916  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1917 
1918  // Verify that the target's LowerReturn behaved as expected.
1919  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1920  "LowerReturn didn't return a valid chain!");
1921 
1922  // Update the DAG with the new chain value resulting from return lowering.
1923  DAG.setRoot(Chain);
1924 }
1925 
1926 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1927 /// created for it, emit nodes to copy the value into the virtual
1928 /// registers.
1930  // Skip empty types
1931  if (V->getType()->isEmptyTy())
1932  return;
1933 
1934  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1935  if (VMI != FuncInfo.ValueMap.end()) {
1936  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1937  CopyValueToVirtualRegister(V, VMI->second);
1938  }
1939 }
1940 
1941 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1942 /// the current basic block, add it to ValueMap now so that we'll get a
1943 /// CopyTo/FromReg.
1945  // No need to export constants.
1946  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1947 
1948  // Already exported?
1949  if (FuncInfo.isExportedInst(V)) return;
1950 
1951  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1952  CopyValueToVirtualRegister(V, Reg);
1953 }
1954 
1956  const BasicBlock *FromBB) {
1957  // The operands of the setcc have to be in this block. We don't know
1958  // how to export them from some other block.
1959  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1960  // Can export from current BB.
1961  if (VI->getParent() == FromBB)
1962  return true;
1963 
1964  // Is already exported, noop.
1965  return FuncInfo.isExportedInst(V);
1966  }
1967 
1968  // If this is an argument, we can export it if the BB is the entry block or
1969  // if it is already exported.
1970  if (isa<Argument>(V)) {
1971  if (FromBB == &FromBB->getParent()->getEntryBlock())
1972  return true;
1973 
1974  // Otherwise, can only export this if it is already exported.
1975  return FuncInfo.isExportedInst(V);
1976  }
1977 
1978  // Otherwise, constants can always be exported.
1979  return true;
1980 }
1981 
1982 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1984 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1985  const MachineBasicBlock *Dst) const {
1986  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1987  const BasicBlock *SrcBB = Src->getBasicBlock();
1988  const BasicBlock *DstBB = Dst->getBasicBlock();
1989  if (!BPI) {
1990  // If BPI is not available, set the default probability as 1 / N, where N is
1991  // the number of successors.
1992  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1993  return BranchProbability(1, SuccSize);
1994  }
1995  return BPI->getEdgeProbability(SrcBB, DstBB);
1996 }
1997 
1998 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1999  MachineBasicBlock *Dst,
2000  BranchProbability Prob) {
2001  if (!FuncInfo.BPI)
2002  Src->addSuccessorWithoutProb(Dst);
2003  else {
2004  if (Prob.isUnknown())
2005  Prob = getEdgeProbability(Src, Dst);
2006  Src->addSuccessor(Dst, Prob);
2007  }
2008 }
2009 
2010 static bool InBlock(const Value *V, const BasicBlock *BB) {
2011  if (const Instruction *I = dyn_cast<Instruction>(V))
2012  return I->getParent() == BB;
2013  return true;
2014 }
2015 
2016 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2017 /// This function emits a branch and is used at the leaves of an OR or an
2018 /// AND operator tree.
2019 void
2021  MachineBasicBlock *TBB,
2022  MachineBasicBlock *FBB,
2023  MachineBasicBlock *CurBB,
2024  MachineBasicBlock *SwitchBB,
2025  BranchProbability TProb,
2026  BranchProbability FProb,
2027  bool InvertCond) {
2028  const BasicBlock *BB = CurBB->getBasicBlock();
2029 
2030  // If the leaf of the tree is a comparison, merge the condition into
2031  // the caseblock.
2032  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2033  // The operands of the cmp have to be in this block. We don't know
2034  // how to export them from some other block. If this is the first block
2035  // of the sequence, no exporting is needed.
2036  if (CurBB == SwitchBB ||
2037  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2038  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2039  ISD::CondCode Condition;
2040  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2041  ICmpInst::Predicate Pred =
2042  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2043  Condition = getICmpCondCode(Pred);
2044  } else {
2045  const FCmpInst *FC = cast<FCmpInst>(Cond);
2046  FCmpInst::Predicate Pred =
2047  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2048  Condition = getFCmpCondCode(Pred);
2049  if (TM.Options.NoNaNsFPMath)
2050  Condition = getFCmpCodeWithoutNaN(Condition);
2051  }
2052 
2053  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2054  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2055  SwitchCases.push_back(CB);
2056  return;
2057  }
2058  }
2059 
2060  // Create a CaseBlock record representing this branch.
2061  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2062  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2063  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2064  SwitchCases.push_back(CB);
2065 }
2066 
2068  MachineBasicBlock *TBB,
2069  MachineBasicBlock *FBB,
2070  MachineBasicBlock *CurBB,
2071  MachineBasicBlock *SwitchBB,
2073  BranchProbability TProb,
2074  BranchProbability FProb,
2075  bool InvertCond) {
2076  // Skip over not part of the tree and remember to invert op and operands at
2077  // next level.
2078  Value *NotCond;
2079  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2080  InBlock(NotCond, CurBB->getBasicBlock())) {
2081  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2082  !InvertCond);
2083  return;
2084  }
2085 
2086  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2087  // Compute the effective opcode for Cond, taking into account whether it needs
2088  // to be inverted, e.g.
2089  // and (not (or A, B)), C
2090  // gets lowered as
2091  // and (and (not A, not B), C)
2092  unsigned BOpc = 0;
2093  if (BOp) {
2094  BOpc = BOp->getOpcode();
2095  if (InvertCond) {
2096  if (BOpc == Instruction::And)
2097  BOpc = Instruction::Or;
2098  else if (BOpc == Instruction::Or)
2099  BOpc = Instruction::And;
2100  }
2101  }
2102 
2103  // If this node is not part of the or/and tree, emit it as a branch.
2104  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
2105  BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
2106  BOp->getParent() != CurBB->getBasicBlock() ||
2107  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
2108  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
2109  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2110  TProb, FProb, InvertCond);
2111  return;
2112  }
2113 
2114  // Create TmpBB after CurBB.
2115  MachineFunction::iterator BBI(CurBB);
2116  MachineFunction &MF = DAG.getMachineFunction();
2118  CurBB->getParent()->insert(++BBI, TmpBB);
2119 
2120  if (Opc == Instruction::Or) {
2121  // Codegen X | Y as:
2122  // BB1:
2123  // jmp_if_X TBB
2124  // jmp TmpBB
2125  // TmpBB:
2126  // jmp_if_Y TBB
2127  // jmp FBB
2128  //
2129 
2130  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2131  // The requirement is that
2132  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2133  // = TrueProb for original BB.
2134  // Assuming the original probabilities are A and B, one choice is to set
2135  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2136  // A/(1+B) and 2B/(1+B). This choice assumes that
2137  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2138  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2139  // TmpBB, but the math is more complicated.
2140 
2141  auto NewTrueProb = TProb / 2;
2142  auto NewFalseProb = TProb / 2 + FProb;
2143  // Emit the LHS condition.
2144  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
2145  NewTrueProb, NewFalseProb, InvertCond);
2146 
2147  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2148  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2149  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2150  // Emit the RHS condition into TmpBB.
2151  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2152  Probs[0], Probs[1], InvertCond);
2153  } else {
2154  assert(Opc == Instruction::And && "Unknown merge op!");
2155  // Codegen X & Y as:
2156  // BB1:
2157  // jmp_if_X TmpBB
2158  // jmp FBB
2159  // TmpBB:
2160  // jmp_if_Y TBB
2161  // jmp FBB
2162  //
2163  // This requires creation of TmpBB after CurBB.
2164 
2165  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2166  // The requirement is that
2167  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2168  // = FalseProb for original BB.
2169  // Assuming the original probabilities are A and B, one choice is to set
2170  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2171  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2172  // TrueProb for BB1 * FalseProb for TmpBB.
2173 
2174  auto NewTrueProb = TProb + FProb / 2;
2175  auto NewFalseProb = FProb / 2;
2176  // Emit the LHS condition.
2177  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
2178  NewTrueProb, NewFalseProb, InvertCond);
2179 
2180  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2181  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2182  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2183  // Emit the RHS condition into TmpBB.
2184  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2185  Probs[0], Probs[1], InvertCond);
2186  }
2187 }
2188 
2189 /// If the set of cases should be emitted as a series of branches, return true.
2190 /// If we should emit this as a bunch of and/or'd together conditions, return
2191 /// false.
2192 bool
2193 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2194  if (Cases.size() != 2) return true;
2195 
2196  // If this is two comparisons of the same values or'd or and'd together, they
2197  // will get folded into a single comparison, so don't emit two blocks.
2198  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2199  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2200  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2201  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2202  return false;
2203  }
2204 
2205  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2206  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2207  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2208  Cases[0].CC == Cases[1].CC &&
2209  isa<Constant>(Cases[0].CmpRHS) &&
2210  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2211  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2212  return false;
2213  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2214  return false;
2215  }
2216 
2217  return true;
2218 }
2219 
2220 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2221  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2222 
2223  // Update machine-CFG edges.
2224  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2225 
2226  if (I.isUnconditional()) {
2227  // Update machine-CFG edges.
2228  BrMBB->addSuccessor(Succ0MBB);
2229 
2230  // If this is not a fall-through branch or optimizations are switched off,
2231  // emit the branch.
2232  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2233  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2234  MVT::Other, getControlRoot(),
2235  DAG.getBasicBlock(Succ0MBB)));
2236 
2237  return;
2238  }
2239 
2240  // If this condition is one of the special cases we handle, do special stuff
2241  // now.
2242  const Value *CondVal = I.getCondition();
2243  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2244 
2245  // If this is a series of conditions that are or'd or and'd together, emit
2246  // this as a sequence of branches instead of setcc's with and/or operations.
2247  // As long as jumps are not expensive, this should improve performance.
2248  // For example, instead of something like:
2249  // cmp A, B
2250  // C = seteq
2251  // cmp D, E
2252  // F = setle
2253  // or C, F
2254  // jnz foo
2255  // Emit:
2256  // cmp A, B
2257  // je foo
2258  // cmp D, E
2259  // jle foo
2260  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2261  Instruction::BinaryOps Opcode = BOp->getOpcode();
2262  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2264  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2265  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2266  Opcode,
2267  getEdgeProbability(BrMBB, Succ0MBB),
2268  getEdgeProbability(BrMBB, Succ1MBB),
2269  /*InvertCond=*/false);
2270  // If the compares in later blocks need to use values not currently
2271  // exported from this block, export them now. This block should always
2272  // be the first entry.
2273  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2274 
2275  // Allow some cases to be rejected.
2276  if (ShouldEmitAsBranches(SwitchCases)) {
2277  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2278  ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2279  ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2280  }
2281 
2282  // Emit the branch for this block.
2283  visitSwitchCase(SwitchCases[0], BrMBB);
2284  SwitchCases.erase(SwitchCases.begin());
2285  return;
2286  }
2287 
2288  // Okay, we decided not to do this, remove any inserted MBB's and clear
2289  // SwitchCases.
2290  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2291  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2292 
2293  SwitchCases.clear();
2294  }
2295  }
2296 
2297  // Create a CaseBlock record representing this branch.
2298  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2299  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2300 
2301  // Use visitSwitchCase to actually insert the fast branch sequence for this
2302  // cond branch.
2303  visitSwitchCase(CB, BrMBB);
2304 }
2305 
2306 /// visitSwitchCase - Emits the necessary code to represent a single node in
2307 /// the binary search tree resulting from lowering a switch instruction.
2309  MachineBasicBlock *SwitchBB) {
2310  SDValue Cond;
2311  SDValue CondLHS = getValue(CB.CmpLHS);
2312  SDLoc dl = CB.DL;
2313 
2314  if (CB.CC == ISD::SETTRUE) {
2315  // Branch or fall through to TrueBB.
2316  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2317  SwitchBB->normalizeSuccProbs();
2318  if (CB.TrueBB != NextBlock(SwitchBB)) {
2319  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2320  DAG.getBasicBlock(CB.TrueBB)));
2321  }
2322  return;
2323  }
2324 
2325  auto &TLI = DAG.getTargetLoweringInfo();
2326  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2327 
2328  // Build the setcc now.
2329  if (!CB.CmpMHS) {
2330  // Fold "(X == true)" to X and "(X == false)" to !X to
2331  // handle common cases produced by branch lowering.
2332  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2333  CB.CC == ISD::SETEQ)
2334  Cond = CondLHS;
2335  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2336  CB.CC == ISD::SETEQ) {
2337  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2338  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2339  } else {
2340  SDValue CondRHS = getValue(CB.CmpRHS);
2341 
2342  // If a pointer's DAG type is larger than its memory type then the DAG
2343  // values are zero-extended. This breaks signed comparisons so truncate
2344  // back to the underlying type before doing the compare.
2345  if (CondLHS.getValueType() != MemVT) {
2346  CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2347  CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2348  }
2349  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2350  }
2351  } else {
2352  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2353 
2354  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2355  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2356 
2357  SDValue CmpOp = getValue(CB.CmpMHS);
2358  EVT VT = CmpOp.getValueType();
2359 
2360  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2361  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2362  ISD::SETLE);
2363  } else {
2364  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2365  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2366  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2367  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2368  }
2369  }
2370 
2371  // Update successor info
2372  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2373  // TrueBB and FalseBB are always different unless the incoming IR is
2374  // degenerate. This only happens when running llc on weird IR.
2375  if (CB.TrueBB != CB.FalseBB)
2376  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2377  SwitchBB->normalizeSuccProbs();
2378 
2379  // If the lhs block is the next block, invert the condition so that we can
2380  // fall through to the lhs instead of the rhs block.
2381  if (CB.TrueBB == NextBlock(SwitchBB)) {
2382  std::swap(CB.TrueBB, CB.FalseBB);
2383  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2384  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2385  }
2386 
2387  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2388  MVT::Other, getControlRoot(), Cond,
2389  DAG.getBasicBlock(CB.TrueBB));
2390 
2391  // Insert the false branch. Do this even if it's a fall through branch,
2392  // this makes it easier to do DAG optimizations which require inverting
2393  // the branch condition.
2394  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2395  DAG.getBasicBlock(CB.FalseBB));
2396 
2397  DAG.setRoot(BrCond);
2398 }
2399 
2400 /// visitJumpTable - Emit JumpTable node in the current MBB
2402  // Emit the code for the jump table
2403  assert(JT.Reg != -1U && "Should lower JT Header first!");
2405  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2406  JT.Reg, PTy);
2407  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2408  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2409  MVT::Other, Index.getValue(1),
2410  Table, Index);
2411  DAG.setRoot(BrJumpTable);
2412 }
2413 
2414 /// visitJumpTableHeader - This function emits necessary code to produce index
2415 /// in the JumpTable from switch case.
2417  JumpTableHeader &JTH,
2418  MachineBasicBlock *SwitchBB) {
2419  SDLoc dl = getCurSDLoc();
2420 
2421  // Subtract the lowest switch case value from the value being switched on.
2422  SDValue SwitchOp = getValue(JTH.SValue);
2423  EVT VT = SwitchOp.getValueType();
2424  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2425  DAG.getConstant(JTH.First, dl, VT));
2426 
2427  // The SDNode we just created, which holds the value being switched on minus
2428  // the smallest case value, needs to be copied to a virtual register so it
2429  // can be used as an index into the jump table in a subsequent basic block.
2430  // This value may be smaller or larger than the target's pointer type, and
2431  // therefore require extension or truncating.
2432  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2433  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2434 
2435  unsigned JumpTableReg =
2436  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2437  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2438  JumpTableReg, SwitchOp);
2439  JT.Reg = JumpTableReg;
2440 
2441  if (!JTH.OmitRangeCheck) {
2442  // Emit the range check for the jump table, and branch to the default block
2443  // for the switch statement if the value being switched on exceeds the
2444  // largest case in the switch.
2445  SDValue CMP = DAG.getSetCC(
2446  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2447  Sub.getValueType()),
2448  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2449 
2450  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2451  MVT::Other, CopyTo, CMP,
2452  DAG.getBasicBlock(JT.Default));
2453 
2454  // Avoid emitting unnecessary branches to the next block.
2455  if (JT.MBB != NextBlock(SwitchBB))
2456  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2457  DAG.getBasicBlock(JT.MBB));
2458 
2459  DAG.setRoot(BrCond);
2460  } else {
2461  // Avoid emitting unnecessary branches to the next block.
2462  if (JT.MBB != NextBlock(SwitchBB))
2463  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2464  DAG.getBasicBlock(JT.MBB)));
2465  else
2466  DAG.setRoot(CopyTo);
2467  }
2468 }
2469 
2470 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2471 /// variable if there exists one.
2473  SDValue &Chain) {
2474  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2475  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2476  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2477  MachineFunction &MF = DAG.getMachineFunction();
2478  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2479  MachineSDNode *Node =
2480  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2481  if (Global) {
2482  MachinePointerInfo MPInfo(Global);
2486  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2487  DAG.setNodeMemRefs(Node, {MemRef});
2488  }
2489  if (PtrTy != PtrMemTy)
2490  return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2491  return SDValue(Node, 0);
2492 }
2493 
2494 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2495 /// tail spliced into a stack protector check success bb.
2496 ///
2497 /// For a high level explanation of how this fits into the stack protector
2498 /// generation see the comment on the declaration of class
2499 /// StackProtectorDescriptor.
2500 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2501  MachineBasicBlock *ParentBB) {
2502 
2503  // First create the loads to the guard/stack slot for the comparison.
2504  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2505  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2506  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2507 
2508  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2509  int FI = MFI.getStackProtectorIndex();
2510 
2511  SDValue Guard;
2512  SDLoc dl = getCurSDLoc();
2513  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2514  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2515  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2516 
2517  // Generate code to load the content of the guard slot.
2518  SDValue GuardVal = DAG.getLoad(
2519  PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2522 
2523  if (TLI.useStackGuardXorFP())
2524  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2525 
2526  // Retrieve guard check function, nullptr if instrumentation is inlined.
2527  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2528  // The target provides a guard check function to validate the guard value.
2529  // Generate a call to that function with the content of the guard slot as
2530  // argument.
2531  FunctionType *FnTy = GuardCheckFn->getFunctionType();
2532  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2533 
2536  Entry.Node = GuardVal;
2537  Entry.Ty = FnTy->getParamType(0);
2538  if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
2539  Entry.IsInReg = true;
2540  Args.push_back(Entry);
2541 
2543  CLI.setDebugLoc(getCurSDLoc())
2544  .setChain(DAG.getEntryNode())
2545  .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2546  getValue(GuardCheckFn), std::move(Args));
2547 
2548  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2549  DAG.setRoot(Result.second);
2550  return;
2551  }
2552 
2553  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2554  // Otherwise, emit a volatile load to retrieve the stack guard value.
2555  SDValue Chain = DAG.getEntryNode();
2556  if (TLI.useLoadStackGuardNode()) {
2557  Guard = getLoadStackGuard(DAG, dl, Chain);
2558  } else {
2559  const Value *IRGuard = TLI.getSDagStackGuard(M);
2560  SDValue GuardPtr = getValue(IRGuard);
2561 
2562  Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2563  MachinePointerInfo(IRGuard, 0), Align,
2565  }
2566 
2567  // Perform the comparison via a subtract/getsetcc.
2568  EVT VT = Guard.getValueType();
2569  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2570 
2571  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2572  *DAG.getContext(),
2573  Sub.getValueType()),
2574  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2575 
2576  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2577  // branch to failure MBB.
2578  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2579  MVT::Other, GuardVal.getOperand(0),
2580  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2581  // Otherwise branch to success MBB.
2582  SDValue Br = DAG.getNode(ISD::BR, dl,
2583  MVT::Other, BrCond,
2584  DAG.getBasicBlock(SPD.getSuccessMBB()));
2585 
2586  DAG.setRoot(Br);
2587 }
2588 
2589 /// Codegen the failure basic block for a stack protector check.
2590 ///
2591 /// A failure stack protector machine basic block consists simply of a call to
2592 /// __stack_chk_fail().
2593 ///
2594 /// For a high level explanation of how this fits into the stack protector
2595 /// generation see the comment on the declaration of class
2596 /// StackProtectorDescriptor.
2597 void
2598 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2599  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2600  SDValue Chain =
2601  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2602  None, false, getCurSDLoc(), false, false).second;
2603  // On PS4, the "return address" must still be within the calling function,
2604  // even if it's at the very end, so emit an explicit TRAP here.
2605  // Passing 'true' for doesNotReturn above won't generate the trap for us.
2606  if (TM.getTargetTriple().isPS4CPU())
2607  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2608 
2609  DAG.setRoot(Chain);
2610 }
2611 
2612 /// visitBitTestHeader - This function emits necessary code to produce value
2613 /// suitable for "bit tests"
2615  MachineBasicBlock *SwitchBB) {
2616  SDLoc dl = getCurSDLoc();
2617 
2618  // Subtract the minimum value
2619  SDValue SwitchOp = getValue(B.SValue);
2620  EVT VT = SwitchOp.getValueType();
2621  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2622  DAG.getConstant(B.First, dl, VT));
2623 
2624  // Check range
2625  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2626  SDValue RangeCmp = DAG.getSetCC(
2627  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2628  Sub.getValueType()),
2629  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2630 
2631  // Determine the type of the test operands.
2632  bool UsePtrType = false;
2633  if (!TLI.isTypeLegal(VT))
2634  UsePtrType = true;
2635  else {
2636  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2637  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2638  // Switch table case range are encoded into series of masks.
2639  // Just use pointer type, it's guaranteed to fit.
2640  UsePtrType = true;
2641  break;
2642  }
2643  }
2644  if (UsePtrType) {
2645  VT = TLI.getPointerTy(DAG.getDataLayout());
2646  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2647  }
2648 
2649  B.RegVT = VT.getSimpleVT();
2650  B.Reg = FuncInfo.CreateReg(B.RegVT);
2651  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2652 
2653  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2654 
2655  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2656  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2657  SwitchBB->normalizeSuccProbs();
2658 
2659  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2660  MVT::Other, CopyTo, RangeCmp,
2661  DAG.getBasicBlock(B.Default));
2662 
2663  // Avoid emitting unnecessary branches to the next block.
2664  if (MBB != NextBlock(SwitchBB))
2665  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2666  DAG.getBasicBlock(MBB));
2667 
2668  DAG.setRoot(BrRange);
2669 }
2670 
2671 /// visitBitTestCase - this function produces one "bit test"
2673  MachineBasicBlock* NextMBB,
2674  BranchProbability BranchProbToNext,
2675  unsigned Reg,
2676  BitTestCase &B,
2677  MachineBasicBlock *SwitchBB) {
2678  SDLoc dl = getCurSDLoc();
2679  MVT VT = BB.RegVT;
2680  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2681  SDValue Cmp;
2682  unsigned PopCount = countPopulation(B.Mask);
2683  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2684  if (PopCount == 1) {
2685  // Testing for a single bit; just compare the shift count with what it
2686  // would need to be to shift a 1 bit in that position.
2687  Cmp = DAG.getSetCC(
2688  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2689  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2690  ISD::SETEQ);
2691  } else if (PopCount == BB.Range) {
2692  // There is only one zero bit in the range, test for it directly.
2693  Cmp = DAG.getSetCC(
2694  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2695  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2696  ISD::SETNE);
2697  } else {
2698  // Make desired shift
2699  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2700  DAG.getConstant(1, dl, VT), ShiftOp);
2701 
2702  // Emit bit tests and jumps
2703  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2704  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2705  Cmp = DAG.getSetCC(
2706  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2707  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2708  }
2709 
2710  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2711  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2712  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2713  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2714  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2715  // one as they are relative probabilities (and thus work more like weights),
2716  // and hence we need to normalize them to let the sum of them become one.
2717  SwitchBB->normalizeSuccProbs();
2718 
2719  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2720  MVT::Other, getControlRoot(),
2721  Cmp, DAG.getBasicBlock(B.TargetBB));
2722 
2723  // Avoid emitting unnecessary branches to the next block.
2724  if (NextMBB != NextBlock(SwitchBB))
2725  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2726  DAG.getBasicBlock(NextMBB));
2727 
2728  DAG.setRoot(BrAnd);
2729 }
2730 
2731 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2732  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2733 
2734  // Retrieve successors. Look through artificial IR level blocks like
2735  // catchswitch for successors.
2736  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2737  const BasicBlock *EHPadBB = I.getSuccessor(1);
2738 
2739  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2740  // have to do anything here to lower funclet bundles.
2742  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2743  "Cannot lower invokes with arbitrary operand bundles yet!");
2744 
2745  const Value *Callee(I.getCalledValue());
2746  const Function *Fn = dyn_cast<Function>(Callee);
2747  if (isa<InlineAsm>(Callee))
2748  visitInlineAsm(&I);
2749  else if (Fn && Fn->isIntrinsic()) {
2750  switch (Fn->getIntrinsicID()) {
2751  default:
2752  llvm_unreachable("Cannot invoke this intrinsic");
2753  case Intrinsic::donothing:
2754  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2755  break;
2756  case Intrinsic::experimental_patchpoint_void:
2757  case Intrinsic::experimental_patchpoint_i64:
2758  visitPatchpoint(&I, EHPadBB);
2759  break;
2760  case Intrinsic::experimental_gc_statepoint:
2761  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2762  break;
2763  case Intrinsic::wasm_rethrow_in_catch: {
2764  // This is usually done in visitTargetIntrinsic, but this intrinsic is
2765  // special because it can be invoked, so we manually lower it to a DAG
2766  // node here.
2768  Ops.push_back(getRoot()); // inchain
2769  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2770  Ops.push_back(
2771  DAG.getTargetConstant(Intrinsic::wasm_rethrow_in_catch, getCurSDLoc(),
2772  TLI.getPointerTy(DAG.getDataLayout())));
2773  SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2774  DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2775  break;
2776  }
2777  }
2779  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2780  // Eventually we will support lowering the @llvm.experimental.deoptimize
2781  // intrinsic, and right now there are no plans to support other intrinsics
2782  // with deopt state.
2783  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2784  } else {
2785  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2786  }
2787 
2788  // If the value of the invoke is used outside of its defining block, make it
2789  // available as a virtual register.
2790  // We already took care of the exported value for the statepoint instruction
2791  // during call to the LowerStatepoint.
2792  if (!isStatepoint(I)) {
2793  CopyToExportRegsIfNeeded(&I);
2794  }
2795 
2797  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2798  BranchProbability EHPadBBProb =
2799  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2801  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2802 
2803  // Update successor info.
2804  addSuccessorWithProb(InvokeMBB, Return);
2805  for (auto &UnwindDest : UnwindDests) {
2806  UnwindDest.first->setIsEHPad();
2807  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2808  }
2809  InvokeMBB->normalizeSuccProbs();
2810 
2811  // Drop into normal successor.
2812  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2813  DAG.getBasicBlock(Return)));
2814 }
2815 
2816 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2817  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2818 
2819  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2820  // have to do anything here to lower funclet bundles.
2822  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2823  "Cannot lower callbrs with arbitrary operand bundles yet!");
2824 
2825  assert(isa<InlineAsm>(I.getCalledValue()) &&
2826  "Only know how to handle inlineasm callbr");
2827  visitInlineAsm(&I);
2828 
2829  // Retrieve successors.
2830  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2831 
2832  // Update successor info.
2833  addSuccessorWithProb(CallBrMBB, Return);
2834  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2835  MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2836  addSuccessorWithProb(CallBrMBB, Target);
2837  }
2838  CallBrMBB->normalizeSuccProbs();
2839 
2840  // Drop into default successor.
2841  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2842  MVT::Other, getControlRoot(),
2843  DAG.getBasicBlock(Return)));
2844 }
2845 
2846 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2847  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2848 }
2849 
2850 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2851  assert(FuncInfo.MBB->isEHPad() &&
2852  "Call to landingpad not in landing pad!");
2853 
2854  // If there aren't registers to copy the values into (e.g., during SjLj
2855  // exceptions), then don't bother to create these DAG nodes.
2856  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2857  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2858  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2859  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2860  return;
2861 
2862  // If landingpad's return type is token type, we don't create DAG nodes
2863  // for its exception pointer and selector value. The extraction of exception
2864  // pointer or selector value from token type landingpads is not currently
2865  // supported.
2866  if (LP.getType()->isTokenTy())
2867  return;
2868 
2870  SDLoc dl = getCurSDLoc();
2871  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2872  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2873 
2874  // Get the two live-in registers as SDValues. The physregs have already been
2875  // copied into virtual registers.
2876  SDValue Ops[2];
2877  if (FuncInfo.ExceptionPointerVirtReg) {
2878  Ops[0] = DAG.getZExtOrTrunc(
2879  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2880  FuncInfo.ExceptionPointerVirtReg,
2881  TLI.getPointerTy(DAG.getDataLayout())),
2882  dl, ValueVTs[0]);
2883  } else {
2884  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2885  }
2886  Ops[1] = DAG.getZExtOrTrunc(
2887  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2888  FuncInfo.ExceptionSelectorVirtReg,
2889  TLI.getPointerTy(DAG.getDataLayout())),
2890  dl, ValueVTs[1]);
2891 
2892  // Merge into one.
2893  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2894  DAG.getVTList(ValueVTs), Ops);
2895  setValue(&LP, Res);
2896 }
2897 
2898 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2899 #ifndef NDEBUG
2900  for (const CaseCluster &CC : Clusters)
2901  assert(CC.Low == CC.High && "Input clusters must be single-case");
2902 #endif
2903 
2904  llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) {
2905  return a.Low->getValue().slt(b.Low->getValue());
2906  });
2907 
2908  // Merge adjacent clusters with the same destination.
2909  const unsigned N = Clusters.size();
2910  unsigned DstIndex = 0;
2911  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2912  CaseCluster &CC = Clusters[SrcIndex];
2913  const ConstantInt *CaseVal = CC.Low;
2914  MachineBasicBlock *Succ = CC.MBB;
2915 
2916  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2917  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2918  // If this case has the same successor and is a neighbour, merge it into
2919  // the previous cluster.
2920  Clusters[DstIndex - 1].High = CaseVal;
2921  Clusters[DstIndex - 1].Prob += CC.Prob;
2922  } else {
2923  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2924  sizeof(Clusters[SrcIndex]));
2925  }
2926  }
2927  Clusters.resize(DstIndex);
2928 }
2929 
2931  MachineBasicBlock *Last) {
2932  // Update JTCases.
2933  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2934  if (JTCases[i].first.HeaderBB == First)
2935  JTCases[i].first.HeaderBB = Last;
2936 
2937  // Update BitTestCases.
2938  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2939  if (BitTestCases[i].Parent == First)
2940  BitTestCases[i].Parent = Last;
2941 }
2942 
2943 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2944  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2945 
2946  // Update machine-CFG edges with unique successors.
2948  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2949  BasicBlock *BB = I.getSuccessor(i);
2950  bool Inserted = Done.insert(BB).second;
2951  if (!Inserted)
2952  continue;
2953 
2954  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2955  addSuccessorWithProb(IndirectBrMBB, Succ);
2956  }
2957  IndirectBrMBB->normalizeSuccProbs();
2958 
2959  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2960  MVT::Other, getControlRoot(),
2961  getValue(I.getAddress())));
2962 }
2963 
2964 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2965  if (!DAG.getTarget().Options.TrapUnreachable)
2966  return;
2967 
2968  // We may be able to ignore unreachable behind a noreturn call.
2970  const BasicBlock &BB = *I.getParent();
2971  if (&I != &BB.front()) {
2973  std::prev(BasicBlock::const_iterator(&I));
2974  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2975  if (Call->doesNotReturn())
2976  return;
2977  }
2978  }
2979  }
2980 
2981  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2982 }
2983 
2984 void SelectionDAGBuilder::visitFSub(const User &I) {
2985  // -0.0 - X --> fneg
2986  Type *Ty = I.getType();
2987  if (isa<Constant>(I.getOperand(0)) &&
2989  SDValue Op2 = getValue(I.getOperand(1));
2990  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2991  Op2.getValueType(), Op2));
2992  return;
2993  }
2994 
2995  visitBinary(I, ISD::FSUB);
2996 }
2997 
2998 /// Checks if the given instruction performs a vector reduction, in which case
2999 /// we have the freedom to alter the elements in the result as long as the
3000 /// reduction of them stays unchanged.
3001 static bool isVectorReductionOp(const User *I) {
3002  const Instruction *Inst = dyn_cast<Instruction>(I);
3003  if (!Inst || !Inst->getType()->isVectorTy())
3004  return false;
3005 
3006  auto OpCode = Inst->getOpcode();
3007  switch (OpCode) {
3008  case Instruction::Add:
3009  case Instruction::Mul:
3010  case Instruction::And:
3011  case Instruction::Or:
3012  case Instruction::Xor:
3013  break;
3014  case Instruction::FAdd:
3015  case Instruction::FMul:
3016  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
3017  if (FPOp->getFastMathFlags().isFast())
3018  break;
3020  default:
3021  return false;
3022  }
3023 
3024  unsigned ElemNum = Inst->getType()->getVectorNumElements();
3025  // Ensure the reduction size is a power of 2.
3026  if (!isPowerOf2_32(ElemNum))
3027  return false;
3028 
3029  unsigned ElemNumToReduce = ElemNum;
3030 
3031  // Do DFS search on the def-use chain from the given instruction. We only
3032  // allow four kinds of operations during the search until we reach the
3033  // instruction that extracts the first element from the vector:
3034  //
3035  // 1. The reduction operation of the same opcode as the given instruction.
3036  //
3037  // 2. PHI node.
3038  //
3039  // 3. ShuffleVector instruction together with a reduction operation that
3040  // does a partial reduction.
3041  //
3042  // 4. ExtractElement that extracts the first element from the vector, and we
3043  // stop searching the def-use chain here.
3044  //
3045  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
3046  // from 1-3 to the stack to continue the DFS. The given instruction is not
3047  // a reduction operation if we meet any other instructions other than those
3048  // listed above.
3049 
3050  SmallVector<const User *, 16> UsersToVisit{Inst};
3052  bool ReduxExtracted = false;
3053 
3054  while (!UsersToVisit.empty()) {
3055  auto User = UsersToVisit.back();
3056  UsersToVisit.pop_back();
3057  if (!Visited.insert(User).second)
3058  continue;
3059 
3060  for (const auto &U : User->users()) {
3061  auto Inst = dyn_cast<Instruction>(U);
3062  if (!Inst)
3063  return false;
3064 
3065  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
3066  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
3067  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
3068  return false;
3069  UsersToVisit.push_back(U);
3070  } else if (const ShuffleVectorInst *ShufInst =
3071  dyn_cast<ShuffleVectorInst>(U)) {
3072  // Detect the following pattern: A ShuffleVector instruction together
3073  // with a reduction that do partial reduction on the first and second
3074  // ElemNumToReduce / 2 elements, and store the result in
3075  // ElemNumToReduce / 2 elements in another vector.
3076 
3077  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
3078  if (ResultElements < ElemNum)
3079  return false;
3080 
3081  if (ElemNumToReduce == 1)
3082  return false;
3083  if (!isa<UndefValue>(U->getOperand(1)))
3084  return false;
3085  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
3086  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
3087  return false;
3088  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
3089  if (ShufInst->getMaskValue(i) != -1)
3090  return false;
3091 
3092  // There is only one user of this ShuffleVector instruction, which
3093  // must be a reduction operation.
3094  if (!U->hasOneUse())
3095  return false;
3096 
3097  auto U2 = dyn_cast<Instruction>(*U->user_begin());
3098  if (!U2 || U2->getOpcode() != OpCode)
3099  return false;
3100 
3101  // Check operands of the reduction operation.
3102  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
3103  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
3104  UsersToVisit.push_back(U2);
3105  ElemNumToReduce /= 2;
3106  } else
3107  return false;
3108  } else if (isa<ExtractElementInst>(U)) {
3109  // At this moment we should have reduced all elements in the vector.
3110  if (ElemNumToReduce != 1)
3111  return false;
3112 
3113  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
3114  if (!Val || !Val->isZero())
3115  return false;
3116 
3117  ReduxExtracted = true;
3118  } else
3119  return false;
3120  }
3121  }
3122  return ReduxExtracted;
3123 }
3124 
3125 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3126  SDNodeFlags Flags;
3127 
3128  SDValue Op = getValue(I.getOperand(0));
3129  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3130  Op, Flags);
3131  setValue(&I, UnNodeValue);
3132 }
3133 
3134 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3135  SDNodeFlags Flags;
3136  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3137  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3138  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3139  }
3140  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
3141  Flags.setExact(ExactOp->isExact());
3142  }
3143  if (isVectorReductionOp(&I)) {
3144  Flags.setVectorReduction(true);
3145  LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
3146  }
3147 
3148  SDValue Op1 = getValue(I.getOperand(0));
3149  SDValue Op2 = getValue(I.getOperand(1));
3150  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3151  Op1, Op2, Flags);
3152  setValue(&I, BinNodeValue);
3153 }
3154 
3155 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3156  SDValue Op1 = getValue(I.getOperand(0));
3157  SDValue Op2 = getValue(I.getOperand(1));
3158 
3159  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3160  Op1.getValueType(), DAG.getDataLayout());
3161 
3162  // Coerce the shift amount to the right type if we can.
3163  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3164  unsigned ShiftSize = ShiftTy.getSizeInBits();
3165  unsigned Op2Size = Op2.getValueSizeInBits();
3166  SDLoc DL = getCurSDLoc();
3167 
3168  // If the operand is smaller than the shift count type, promote it.
3169  if (ShiftSize > Op2Size)
3170  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3171 
3172  // If the operand is larger than the shift count type but the shift
3173  // count type has enough bits to represent any shift value, truncate
3174  // it now. This is a common case and it exposes the truncate to
3175  // optimization early.
3176  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3177  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3178  // Otherwise we'll need to temporarily settle for some other convenient
3179  // type. Type legalization will make adjustments once the shiftee is split.
3180  else
3181  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3182  }
3183 
3184  bool nuw = false;
3185  bool nsw = false;
3186  bool exact = false;
3187 
3188  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3189 
3190  if (const OverflowingBinaryOperator *OFBinOp =
3191  dyn_cast<const OverflowingBinaryOperator>(&I)) {
3192  nuw = OFBinOp->hasNoUnsignedWrap();
3193  nsw = OFBinOp->hasNoSignedWrap();
3194  }
3195  if (const PossiblyExactOperator *ExactOp =
3196  dyn_cast<const PossiblyExactOperator>(&I))
3197  exact = ExactOp->isExact();
3198  }
3199  SDNodeFlags Flags;
3200  Flags.setExact(exact);
3201  Flags.setNoSignedWrap(nsw);
3202  Flags.setNoUnsignedWrap(nuw);
3203  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3204  Flags);
3205  setValue(&I, Res);
3206 }
3207 
3208 void SelectionDAGBuilder::visitSDiv(const User &I) {
3209  SDValue Op1 = getValue(I.getOperand(0));
3210  SDValue Op2 = getValue(I.getOperand(1));
3211 
3212  SDNodeFlags Flags;
3213  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3214  cast<PossiblyExactOperator>(&I)->isExact());
3215  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3216  Op2, Flags));
3217 }
3218 
3219 void SelectionDAGBuilder::visitICmp(const User &I) {
3221  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3222  predicate = IC->getPredicate();
3223  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3224  predicate = ICmpInst::Predicate(IC->getPredicate());
3225  SDValue Op1 = getValue(I.getOperand(0));
3226  SDValue Op2 = getValue(I.getOperand(1));
3227  ISD::CondCode Opcode = getICmpCondCode(predicate);
3228 
3229  auto &TLI = DAG.getTargetLoweringInfo();
3230  EVT MemVT =
3231  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3232 
3233  // If a pointer's DAG type is larger than its memory type then the DAG values
3234  // are zero-extended. This breaks signed comparisons so truncate back to the
3235  // underlying type before doing the compare.
3236  if (Op1.getValueType() != MemVT) {
3237  Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3238  Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3239  }
3240 
3241  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3242  I.getType());
3243  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3244 }
3245 
3246 void SelectionDAGBuilder::visitFCmp(const User &I) {
3248  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3249  predicate = FC->getPredicate();
3250  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3251  predicate = FCmpInst::Predicate(FC->getPredicate());
3252  SDValue Op1 = getValue(I.getOperand(0));
3253  SDValue Op2 = getValue(I.getOperand(1));
3254 
3255  ISD::CondCode Condition = getFCmpCondCode(predicate);
3256  auto *FPMO = dyn_cast<FPMathOperator>(&I);
3257  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
3258  Condition = getFCmpCodeWithoutNaN(Condition);
3259 
3260  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3261  I.getType());
3262  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3263 }
3264 
3265 // Check if the condition of the select has one use or two users that are both
3266 // selects with the same condition.
3267 static bool hasOnlySelectUsers(const Value *Cond) {
3268  return llvm::all_of(Cond->users(), [](const Value *V) {
3269  return isa<SelectInst>(V);
3270  });
3271 }
3272 
3273 void SelectionDAGBuilder::visitSelect(const User &I) {
3276  ValueVTs);
3277  unsigned NumValues = ValueVTs.size();
3278  if (NumValues == 0) return;
3279 
3280  SmallVector<SDValue, 4> Values(NumValues);
3281  SDValue Cond = getValue(I.getOperand(0));
3282  SDValue LHSVal = getValue(I.getOperand(1));
3283  SDValue RHSVal = getValue(I.getOperand(2));
3284  auto BaseOps = {Cond};
3285  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
3287 
3288  bool IsUnaryAbs = false;
3289 
3290  // Min/max matching is only viable if all output VTs are the same.
3291  if (is_splat(ValueVTs)) {
3292  EVT VT = ValueVTs[0];
3293  LLVMContext &Ctx = *DAG.getContext();
3294  auto &TLI = DAG.getTargetLoweringInfo();
3295 
3296  // We care about the legality of the operation after it has been type
3297  // legalized.
3298  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
3299  VT != TLI.getTypeToTransformTo(Ctx, VT))
3300  VT = TLI.getTypeToTransformTo(Ctx, VT);
3301 
3302  // If the vselect is legal, assume we want to leave this as a vector setcc +
3303  // vselect. Otherwise, if this is going to be scalarized, we want to see if
3304  // min/max is legal on the scalar type.
3305  bool UseScalarMinMax = VT.isVector() &&
3306  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3307 
3308  Value *LHS, *RHS;
3309  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3311  switch (SPR.Flavor) {
3312  case SPF_UMAX: Opc = ISD::UMAX; break;
3313  case SPF_UMIN: Opc = ISD::UMIN; break;
3314  case SPF_SMAX: Opc = ISD::SMAX; break;
3315  case SPF_SMIN: Opc = ISD::SMIN; break;
3316  case SPF_FMINNUM:
3317  switch (SPR.NaNBehavior) {
3318  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3319  case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3320  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3321  case SPNB_RETURNS_ANY: {
3322  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3323  Opc = ISD::FMINNUM;
3324  else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3325  Opc = ISD::FMINIMUM;
3326  else if (UseScalarMinMax)
3327  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3329  break;
3330  }
3331  }
3332  break;
3333  case SPF_FMAXNUM:
3334  switch (SPR.NaNBehavior) {
3335  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3336  case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3337  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3338  case SPNB_RETURNS_ANY:
3339 
3340  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3341  Opc = ISD::FMAXNUM;
3342  else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3343  Opc = ISD::FMAXIMUM;
3344  else if (UseScalarMinMax)
3345  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3347  break;
3348  }
3349  break;
3350  case SPF_ABS:
3351  IsUnaryAbs = true;
3352  Opc = ISD::ABS;
3353  break;
3354  case SPF_NABS:
3355  // TODO: we need to produce sub(0, abs(X)).
3356  default: break;
3357  }
3358 
3359  if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3360  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3361  (UseScalarMinMax &&
3362  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3363  // If the underlying comparison instruction is used by any other
3364  // instruction, the consumed instructions won't be destroyed, so it is
3365  // not profitable to convert to a min/max.
3366  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3367  OpCode = Opc;
3368  LHSVal = getValue(LHS);
3369  RHSVal = getValue(RHS);
3370  BaseOps = {};
3371  }
3372 
3373  if (IsUnaryAbs) {
3374  OpCode = Opc;
3375  LHSVal = getValue(LHS);
3376  BaseOps = {};
3377  }
3378  }
3379 
3380  if (IsUnaryAbs) {
3381  for (unsigned i = 0; i != NumValues; ++i) {
3382  Values[i] =
3383  DAG.getNode(OpCode, getCurSDLoc(),
3384  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),
3385  SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3386  }
3387  } else {
3388  for (unsigned i = 0; i != NumValues; ++i) {
3389  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3390  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3391  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3392  Values[i] = DAG.getNode(
3393  OpCode, getCurSDLoc(),
3394  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
3395  }
3396  }
3397 
3398  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3399  DAG.getVTList(ValueVTs), Values));
3400 }
3401 
3402 void SelectionDAGBuilder::visitTrunc(const User &I) {
3403  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3404  SDValue N = getValue(I.getOperand(0));
3405  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3406  I.getType());
3407  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3408 }
3409 
3410 void SelectionDAGBuilder::visitZExt(const User &I) {
3411  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3412  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3413  SDValue N = getValue(I.getOperand(0));
3414  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3415  I.getType());
3416  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3417 }
3418 
3419 void SelectionDAGBuilder::visitSExt(const User &I) {
3420  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3421  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3422  SDValue N = getValue(I.getOperand(0));
3423  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3424  I.getType());
3425  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3426 }
3427 
3428 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3429  // FPTrunc is never a no-op cast, no need to check
3430  SDValue N = getValue(I.getOperand(0));
3431  SDLoc dl = getCurSDLoc();
3432  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3433  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3434  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3435  DAG.getTargetConstant(
3436  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3437 }
3438 
3439 void SelectionDAGBuilder::visitFPExt(const User &I) {
3440  // FPExt is never a no-op cast, no need to check
3441  SDValue N = getValue(I.getOperand(0));
3442  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3443  I.getType());
3444  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3445 }
3446 
3447 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3448  // FPToUI is never a no-op cast, no need to check
3449  SDValue N = getValue(I.getOperand(0));
3450  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3451  I.getType());
3452  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3453 }
3454 
3455 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3456  // FPToSI is never a no-op cast, no need to check
3457  SDValue N = getValue(I.getOperand(0));
3458  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3459  I.getType());
3460  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3461 }
3462 
3463 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3464  // UIToFP is never a no-op cast, no need to check
3465  SDValue N = getValue(I.getOperand(0));
3466  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3467  I.getType());
3468  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3469 }
3470 
3471 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3472  // SIToFP is never a no-op cast, no need to check
3473  SDValue N = getValue(I.getOperand(0));
3474  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3475  I.getType());
3476  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3477 }
3478 
3479 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3480  // What to do depends on the size of the integer and the size of the pointer.
3481  // We can either truncate, zero extend, or no-op, accordingly.
3482  SDValue N = getValue(I.getOperand(0));
3483  auto &TLI = DAG.getTargetLoweringInfo();
3484  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3485  I.getType());
3486  EVT PtrMemVT =
3487  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3488  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3489  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3490  setValue(&I, N);
3491 }
3492 
3493 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3494  // What to do depends on the size of the integer and the size of the pointer.
3495  // We can either truncate, zero extend, or no-op, accordingly.
3496  SDValue N = getValue(I.getOperand(0));
3497  auto &TLI = DAG.getTargetLoweringInfo();
3498  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3499  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3500  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3501  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3502  setValue(&I, N);
3503 }
3504 
3505 void SelectionDAGBuilder::visitBitCast(const User &I) {
3506  SDValue N = getValue(I.getOperand(0));
3507  SDLoc dl = getCurSDLoc();
3508  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3509  I.getType());
3510 
3511  // BitCast assures us that source and destination are the same size so this is
3512  // either a BITCAST or a no-op.
3513  if (DestVT != N.getValueType())
3514  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3515  DestVT, N)); // convert types.
3516  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3517  // might fold any kind of constant expression to an integer constant and that
3518  // is not what we are looking for. Only recognize a bitcast of a genuine
3519  // constant integer as an opaque constant.
3520  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3521  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3522  /*isOpaque*/true));
3523  else
3524  setValue(&I, N); // noop cast.
3525 }
3526 
3527 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3528  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3529  const Value *SV = I.getOperand(0);
3530  SDValue N = getValue(SV);
3531  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3532 
3533  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3534  unsigned DestAS = I.getType()->getPointerAddressSpace();
3535 
3536  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3537  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3538 
3539  setValue(&I, N);
3540 }
3541 
3542 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3543  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3544  SDValue InVec = getValue(I.getOperand(0));
3545  SDValue InVal = getValue(I.getOperand(1));
3546  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3547  TLI.getVectorIdxTy(DAG.getDataLayout()));
3548  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3549  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3550  InVec, InVal, InIdx));
3551 }
3552 
3553 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3554  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3555  SDValue InVec = getValue(I.getOperand(0));
3556  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3557  TLI.getVectorIdxTy(DAG.getDataLayout()));
3558  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3559  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3560  InVec, InIdx));
3561 }
3562 
3563 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3564  SDValue Src1 = getValue(I.getOperand(0));
3565  SDValue Src2 = getValue(I.getOperand(1));
3566  SDLoc DL = getCurSDLoc();
3567 
3569  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3570  unsigned MaskNumElts = Mask.size();
3571 
3572  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3573  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3574  EVT SrcVT = Src1.getValueType();
3575  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3576 
3577  if (SrcNumElts == MaskNumElts) {
3578  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3579  return;
3580  }
3581 
3582  // Normalize the shuffle vector since mask and vector length don't match.
3583  if (SrcNumElts < MaskNumElts) {
3584  // Mask is longer than the source vectors. We can use concatenate vector to
3585  // make the mask and vectors lengths match.
3586 
3587  if (MaskNumElts % SrcNumElts == 0) {
3588  // Mask length is a multiple of the source vector length.
3589  // Check if the shuffle is some kind of concatenation of the input
3590  // vectors.
3591  unsigned NumConcat = MaskNumElts / SrcNumElts;
3592  bool IsConcat = true;
3593  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3594  for (unsigned i = 0; i != MaskNumElts; ++i) {
3595  int Idx = Mask[i];
3596  if (Idx < 0)
3597  continue;
3598  // Ensure the indices in each SrcVT sized piece are sequential and that
3599  // the same source is used for the whole piece.
3600  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3601  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3602  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3603  IsConcat = false;
3604  break;
3605  }
3606  // Remember which source this index came from.
3607  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3608  }
3609 
3610  // The shuffle is concatenating multiple vectors together. Just emit
3611  // a CONCAT_VECTORS operation.
3612  if (IsConcat) {
3613  SmallVector<SDValue, 8> ConcatOps;
3614  for (auto Src : ConcatSrcs) {
3615  if (Src < 0)
3616  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3617  else if (Src == 0)
3618  ConcatOps.push_back(Src1);
3619  else
3620  ConcatOps.push_back(Src2);
3621  }
3622  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3623  return;
3624  }
3625  }
3626 
3627  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3628  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3629  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3630  PaddedMaskNumElts);
3631 
3632  // Pad both vectors with undefs to make them the same length as the mask.
3633  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3634 
3635  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3636  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3637  MOps1[0] = Src1;
3638  MOps2[0] = Src2;
3639 
3640  Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3641  Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3642 
3643  // Readjust mask for new input vector length.
3644  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3645  for (unsigned i = 0; i != MaskNumElts; ++i) {
3646  int Idx = Mask[i];
3647  if (Idx >= (int)SrcNumElts)
3648  Idx -= SrcNumElts - PaddedMaskNumElts;
3649  MappedOps[i] = Idx;
3650  }
3651 
3652  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3653 
3654  // If the concatenated vector was padded, extract a subvector with the
3655  // correct number of elements.
3656  if (MaskNumElts != PaddedMaskNumElts)
3657  Result = DAG.getNode(
3658  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3659  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3660 
3661  setValue(&I, Result);
3662  return;
3663  }
3664 
3665  if (SrcNumElts > MaskNumElts) {
3666  // Analyze the access pattern of the vector to see if we can extract
3667  // two subvectors and do the shuffle.
3668  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3669  bool CanExtract = true;
3670  for (int Idx : Mask) {
3671  unsigned Input = 0;
3672  if (Idx < 0)
3673  continue;
3674 
3675  if (Idx >= (int)SrcNumElts) {
3676  Input = 1;
3677  Idx -= SrcNumElts;
3678  }
3679 
3680  // If all the indices come from the same MaskNumElts sized portion of
3681  // the sources we can use extract. Also make sure the extract wouldn't
3682  // extract past the end of the source.
3683  int NewStartIdx = alignDown(Idx, MaskNumElts);
3684  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3685  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3686  CanExtract = false;
3687  // Make sure we always update StartIdx as we use it to track if all
3688  // elements are undef.
3689  StartIdx[Input] = NewStartIdx;
3690  }
3691 
3692  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3693  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3694  return;
3695  }
3696  if (CanExtract) {
3697  // Extract appropriate subvector and generate a vector shuffle
3698  for (unsigned Input = 0; Input < 2; ++Input) {
3699  SDValue &Src = Input == 0 ? Src1 : Src2;
3700  if (StartIdx[Input] < 0)
3701  Src = DAG.getUNDEF(VT);
3702  else {
3703  Src = DAG.getNode(
3704  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3705  DAG.getConstant(StartIdx[Input], DL,
3706  TLI.getVectorIdxTy(DAG.getDataLayout())));
3707  }
3708  }
3709 
3710  // Calculate new mask.
3711  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3712  for (int &Idx : MappedOps) {
3713  if (Idx >= (int)SrcNumElts)
3714  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3715  else if (Idx >= 0)
3716  Idx -= StartIdx[0];
3717  }
3718 
3719  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3720  return;
3721  }
3722  }
3723 
3724  // We can't use either concat vectors or extract subvectors so fall back to
3725  // replacing the shuffle with extract and build vector.
3726  // to insert and build vector.
3727  EVT EltVT = VT.getVectorElementType();
3728  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3730  for (int Idx : Mask) {
3731  SDValue Res;
3732 
3733  if (Idx < 0) {
3734  Res = DAG.getUNDEF(EltVT);
3735  } else {
3736  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3737  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3738 
3739  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3740  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3741  }
3742 
3743  Ops.push_back(Res);
3744  }
3745 
3746  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3747 }
3748 
3749 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3750  ArrayRef<unsigned> Indices;
3751  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3752  Indices = IV->getIndices();
3753  else
3754  Indices = cast<ConstantExpr>(&I)->getIndices();
3755 
3756  const Value *Op0 = I.getOperand(0);
3757  const Value *Op1 = I.getOperand(1);
3758  Type *AggTy = I.getType();
3759  Type *ValTy = Op1->getType();
3760  bool IntoUndef = isa<UndefValue>(Op0);
3761  bool FromUndef = isa<UndefValue>(Op1);
3762 
3763  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3764 
3765  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3766  SmallVector<EVT, 4> AggValueVTs;
3767  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3768  SmallVector<EVT, 4> ValValueVTs;
3769  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3770 
3771  unsigned NumAggValues = AggValueVTs.size();
3772  unsigned NumValValues = ValValueVTs.size();
3773  SmallVector<SDValue, 4> Values(NumAggValues);
3774 
3775  // Ignore an insertvalue that produces an empty object
3776  if (!NumAggValues) {
3777  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3778  return;
3779  }
3780 
3781  SDValue Agg = getValue(Op0);
3782  unsigned i = 0;
3783  // Copy the beginning value(s) from the original aggregate.
3784  for (; i != LinearIndex; ++i)
3785  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3786  SDValue(Agg.getNode(), Agg.getResNo() + i);
3787  // Copy values from the inserted value(s).
3788  if (NumValValues) {
3789  SDValue Val = getValue(Op1);
3790  for (; i != LinearIndex + NumValValues; ++i)
3791  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3792  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3793  }
3794  // Copy remaining value(s) from the original aggregate.
3795  for (; i != NumAggValues; ++i)
3796  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3797  SDValue(Agg.getNode(), Agg.getResNo() + i);
3798 
3799  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3800  DAG.getVTList(AggValueVTs), Values));
3801 }
3802 
3803 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3804  ArrayRef<unsigned> Indices;
3805  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3806  Indices = EV->getIndices();
3807  else
3808  Indices = cast<ConstantExpr>(&I)->getIndices();
3809 
3810  const Value *Op0 = I.getOperand(0);
3811  Type *AggTy = Op0->getType();
3812  Type *ValTy = I.getType();
3813  bool OutOfUndef = isa<UndefValue>(Op0);
3814 
3815  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3816 
3817  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3818  SmallVector<EVT, 4> ValValueVTs;
3819  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3820 
3821  unsigned NumValValues = ValValueVTs.size();
3822 
3823  // Ignore a extractvalue that produces an empty object
3824  if (!NumValValues) {
3825  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3826  return;
3827  }
3828 
3829  SmallVector<SDValue, 4> Values(NumValValues);
3830 
3831  SDValue Agg = getValue(Op0);
3832  // Copy out the selected value(s).
3833  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3834  Values[i - LinearIndex] =
3835  OutOfUndef ?
3836  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3837  SDValue(Agg.getNode(), Agg.getResNo() + i);
3838 
3839  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3840  DAG.getVTList(ValValueVTs), Values));
3841 }
3842 
3843 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3844  Value *Op0 = I.getOperand(0);
3845  // Note that the pointer operand may be a vector of pointers. Take the scalar
3846  // element which holds a pointer.
3847  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3848  SDValue N = getValue(Op0);
3849  SDLoc dl = getCurSDLoc();
3850  auto &TLI = DAG.getTargetLoweringInfo();
3851  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3852  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3853 
3854  // Normalize Vector GEP - all scalar operands should be converted to the
3855  // splat vector.
3856  unsigned VectorWidth = I.getType()->isVectorTy() ?
3857  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3858 
3859  if (VectorWidth && !N.getValueType().isVector()) {
3860  LLVMContext &Context = *DAG.getContext();
3861  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3862  N = DAG.getSplatBuildVector(VT, dl, N);
3863  }
3864 
3865  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3866  GTI != E; ++GTI) {
3867  const Value *Idx = GTI.getOperand();
3868  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3869  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3870  if (Field) {
3871  // N = N + Offset
3872  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3873 
3874  // In an inbounds GEP with an offset that is nonnegative even when
3875  // interpreted as signed, assume there is no unsigned overflow.
3876  SDNodeFlags Flags;
3877  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3878  Flags.setNoUnsignedWrap(true);
3879 
3880  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3881  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3882  }
3883  } else {
3884  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3885  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3886  APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3887 
3888  // If this is a scalar constant or a splat vector of constants,
3889  // handle it quickly.
3890  const auto *CI = dyn_cast<ConstantInt>(Idx);
3891  if (!CI && isa<ConstantDataVector>(Idx) &&
3892  cast<ConstantDataVector>(Idx)->getSplatValue())
3893  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3894 
3895  if (CI) {
3896  if (CI->isZero())
3897  continue;
3898  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3899  LLVMContext &Context = *DAG.getContext();
3900  SDValue OffsVal = VectorWidth ?
3901  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3902  DAG.getConstant(Offs, dl, IdxTy);
3903 
3904  // In an inbouds GEP with an offset that is nonnegative even when
3905  // interpreted as signed, assume there is no unsigned overflow.
3906  SDNodeFlags Flags;
3907  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3908  Flags.setNoUnsignedWrap(true);
3909 
3910  OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3911 
3912  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3913  continue;
3914  }
3915 
3916  // N = N + Idx * ElementSize;
3917  SDValue IdxN = getValue(Idx);
3918 
3919  if (!IdxN.getValueType().isVector() && VectorWidth) {
3920  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3921  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3922  }
3923 
3924  // If the index is smaller or larger than intptr_t, truncate or extend
3925  // it.
3926  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3927 
3928  // If this is a multiply by a power of two, turn it into a shl
3929  // immediately. This is a very common case.
3930  if (ElementSize != 1) {
3931  if (ElementSize.isPowerOf2()) {
3932  unsigned Amt = ElementSize.logBase2();
3933  IdxN = DAG.getNode(ISD::SHL, dl,
3934  N.getValueType(), IdxN,
3935  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3936  } else {
3937  SDValue Scale = DAG.getConstant(ElementSize.getZExtValue(), dl,
3938  IdxN.getValueType());
3939  IdxN = DAG.getNode(ISD::MUL, dl,
3940  N.getValueType(), IdxN, Scale);
3941  }
3942  }
3943 
3944  N = DAG.getNode(ISD::ADD, dl,
3945  N.getValueType(), N, IdxN);
3946  }
3947  }
3948 
3949  if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3950  N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3951 
3952  setValue(&I, N);
3953 }
3954 
3955 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3956  // If this is a fixed sized alloca in the entry block of the function,
3957  // allocate it statically on the stack.
3958  if (FuncInfo.StaticAllocaMap.count(&I))
3959  return; // getValue will auto-populate this.
3960 
3961  SDLoc dl = getCurSDLoc();
3962  Type *Ty = I.getAllocatedType();
3963  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3964  auto &DL = DAG.getDataLayout();
3965  uint64_t TySize = DL.getTypeAllocSize(Ty);
3966  unsigned Align =
3967  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3968 
3969  SDValue AllocSize = getValue(I.getArraySize());
3970 
3971  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3972  if (AllocSize.getValueType() != IntPtr)
3973  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3974 
3975  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3976  AllocSize,
3977  DAG.getConstant(TySize, dl, IntPtr));
3978 
3979  // Handle alignment. If the requested alignment is less than or equal to
3980  // the stack alignment, ignore it. If the size is greater than or equal to
3981  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3982  unsigned StackAlign =
3984  if (Align <= StackAlign)
3985  Align = 0;
3986 
3987  // Round the size of the allocation up to the stack alignment size
3988  // by add SA-1 to the size. This doesn't overflow because we're computing
3989  // an address inside an alloca.
3990  SDNodeFlags Flags;
3991  Flags.setNoUnsignedWrap(true);
3992  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3993  DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3994 
3995  // Mask out the low bits for alignment purposes.
3996  AllocSize =
3997  DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3998  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3999 
4000  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
4001  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4002  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4003  setValue(&I, DSA);
4004  DAG.setRoot(DSA.getValue(1));
4005 
4006  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4007 }
4008 
4009 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4010  if (I.isAtomic())
4011  return visitAtomicLoad(I);
4012 
4013  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4014  const Value *SV = I.getOperand(0);
4015  if (TLI.supportSwiftError()) {
4016  // Swifterror values can come from either a function parameter with
4017  // swifterror attribute or an alloca with swifterror attribute.
4018  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4019  if (Arg->hasSwiftErrorAttr())
4020  return visitLoadFromSwiftError(I);
4021  }
4022 
4023  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4024  if (Alloca->isSwiftError())
4025  return visitLoadFromSwiftError(I);
4026  }
4027  }
4028 
4029  SDValue Ptr = getValue(SV);
4030 
4031  Type *Ty = I.getType();
4032 
4033  bool isVolatile = I.isVolatile();
4034  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
4035  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
4036  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
4037  unsigned Alignment = I.getAlignment();
4038 
4039  AAMDNodes AAInfo;
4040  I.getAAMetadata(AAInfo);
4041  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4042 
4043  SmallVector<EVT, 4> ValueVTs, MemVTs;
4045  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4046  unsigned NumValues = ValueVTs.size();
4047  if (NumValues == 0)
4048  return;
4049 
4050  SDValue Root;
4051  bool ConstantMemory = false;
4052  if (isVolatile || NumValues > MaxParallelChains)
4053  // Serialize volatile loads with other side effects.
4054  Root = getRoot();
4055  else if (AA &&
4056  AA->pointsToConstantMemory(MemoryLocation(
4057  SV,
4059  AAInfo))) {
4060  // Do not serialize (non-volatile) loads of constant memory with anything.
4061  Root = DAG.getEntryNode();
4062  ConstantMemory = true;
4063  } else {
4064  // Do not serialize non-volatile loads against each other.
4065  Root = DAG.getRoot();
4066  }
4067 
4068  SDLoc dl = getCurSDLoc();
4069 
4070  if (isVolatile)
4071  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4072 
4073  // An aggregate load cannot wrap around the address space, so offsets to its
4074  // parts don't wrap either.
4075  SDNodeFlags Flags;
4076  Flags.setNoUnsignedWrap(true);
4077 
4078  SmallVector<SDValue, 4> Values(NumValues);
4079  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4080  EVT PtrVT = Ptr.getValueType();
4081  unsigned ChainI = 0;
4082  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4083  // Serializing loads here may result in excessive register pressure, and
4084  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4085  // could recover a bit by hoisting nodes upward in the chain by recognizing
4086  // they are side-effect free or do not alias. The optimizer should really
4087  // avoid this case by converting large object/array copies to llvm.memcpy
4088  // (MaxParallelChains should always remain as failsafe).
4089  if (ChainI == MaxParallelChains) {
4090  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4091  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4092  makeArrayRef(Chains.data(), ChainI));
4093  Root = Chain;
4094  ChainI = 0;
4095  }
4096  SDValue A = DAG.getNode(ISD::ADD, dl,
4097  PtrVT, Ptr,
4098  DAG.getConstant(Offsets[i], dl, PtrVT),
4099  Flags);
4100  auto MMOFlags = MachineMemOperand::MONone;
4101  if (isVolatile)
4102  MMOFlags |= MachineMemOperand::MOVolatile;
4103  if (isNonTemporal)
4105  if (isInvariant)
4106  MMOFlags |= MachineMemOperand::MOInvariant;
4107  if (isDereferenceable)
4109  MMOFlags |= TLI.getMMOFlags(I);
4110 
4111  SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4112  MachinePointerInfo(SV, Offsets[i]), Alignment,
4113  MMOFlags, AAInfo, Ranges);
4114  Chains[ChainI] = L.getValue(1);
4115 
4116  if (MemVTs[i] != ValueVTs[i])
4117  L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4118 
4119  Values[i] = L;
4120  }
4121 
4122  if (!ConstantMemory) {
4123  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4124  makeArrayRef(Chains.data(), ChainI));
4125  if (isVolatile)
4126  DAG.setRoot(Chain);
4127  else
4128  PendingLoads.push_back(Chain);
4129  }
4130 
4131  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4132  DAG.getVTList(ValueVTs), Values));
4133 }
4134 
4135 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4137  "call visitStoreToSwiftError when backend supports swifterror");
4138 
4141  const Value *SrcV = I.getOperand(0);
4143  SrcV->getType(), ValueVTs, &Offsets);
4144  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4145  "expect a single EVT for swifterror");
4146 
4147  SDValue Src = getValue(SrcV);
4148  // Create a virtual register, then update the virtual register.
4149  unsigned VReg; bool CreatedVReg;
4150  std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
4151  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4152  // Chain can be getRoot or getControlRoot.
4153  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4154  SDValue(Src.getNode(), Src.getResNo()));
4155  DAG.setRoot(CopyNode);
4156  if (CreatedVReg)
4157  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
4158 }
4159 
4160 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4162  "call visitLoadFromSwiftError when backend supports swifterror");
4163 
4164  assert(!I.isVolatile() &&
4165  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
4167  "Support volatile, non temporal, invariant for load_from_swift_error");
4168 
4169  const Value *SV = I.getOperand(0);
4170  Type *Ty = I.getType();
4171  AAMDNodes AAInfo;
4172  I.getAAMetadata(AAInfo);
4173  assert(
4174  (!AA ||
4175  !AA->pointsToConstantMemory(MemoryLocation(
4177  AAInfo))) &&
4178  "load_from_swift_error should not be constant memory");
4179 
4183  ValueVTs, &Offsets);
4184  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4185  "expect a single EVT for swifterror");
4186 
4187  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4188  SDValue L = DAG.getCopyFromReg(
4189  getRoot(), getCurSDLoc(),
4190  FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
4191  ValueVTs[0]);
4192 
4193  setValue(&I, L);
4194 }
4195 
4196 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4197  if (I.isAtomic())
4198  return visitAtomicStore(I);
4199 
4200  const Value *SrcV = I.getOperand(0);
4201  const Value *PtrV = I.getOperand(1);
4202 
4203  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4204  if (TLI.supportSwiftError()) {
4205  // Swifterror values can come from either a function parameter with
4206  // swifterror attribute or an alloca with swifterror attribute.
4207  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4208  if (Arg->hasSwiftErrorAttr())
4209  return visitStoreToSwiftError(I);
4210  }
4211 
4212  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4213  if (Alloca->isSwiftError())
4214  return visitStoreToSwiftError(I);
4215  }
4216  }
4217 
4218  SmallVector<EVT, 4> ValueVTs, MemVTs;
4221  SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4222  unsigned NumValues = ValueVTs.size();
4223  if (NumValues == 0)
4224  return;
4225 
4226  // Get the lowered operands. Note that we do this after
4227  // checking if NumResults is zero, because with zero results
4228  // the operands won't have values in the map.
4229  SDValue Src = getValue(SrcV);
4230  SDValue Ptr = getValue(PtrV);
4231 
4232  SDValue Root = getRoot();
4233  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4234  SDLoc dl = getCurSDLoc();
4235  EVT PtrVT = Ptr.getValueType();
4236  unsigned Alignment = I.getAlignment();
4237  AAMDNodes AAInfo;
4238  I.getAAMetadata(AAInfo);
4239 
4240  auto MMOFlags = MachineMemOperand::MONone;
4241  if (I.isVolatile())
4242  MMOFlags |= MachineMemOperand::MOVolatile;
4243  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
4245  MMOFlags |= TLI.getMMOFlags(I);
4246 
4247  // An aggregate load cannot wrap around the address space, so offsets to its
4248  // parts don't wrap either.
4249  SDNodeFlags Flags;
4250  Flags.setNoUnsignedWrap(true);
4251 
4252  unsigned ChainI = 0;
4253  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4254  // See visitLoad comments.
4255  if (ChainI == MaxParallelChains) {
4256  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4257  makeArrayRef(Chains.data(), ChainI));
4258  Root = Chain;
4259  ChainI = 0;
4260  }
4261  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
4262  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
4263  SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4264  if (MemVTs[i] != ValueVTs[i])
4265  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4266  SDValue St =
4267  DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4268  Alignment, MMOFlags, AAInfo);
4269  Chains[ChainI] = St;
4270  }
4271 
4272  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4273  makeArrayRef(Chains.data(), ChainI));
4274  DAG.setRoot(StoreNode);
4275 }
4276 
4277 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4278  bool IsCompressing) {
4279  SDLoc sdl = getCurSDLoc();
4280 
4281  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4282  unsigned& Alignment) {
4283  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4284  Src0 = I.getArgOperand(0);
4285  Ptr = I.getArgOperand(1);
4286  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
4287  Mask = I.getArgOperand(3);
4288  };
4289  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4290  unsigned& Alignment) {
4291  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4292  Src0 = I.getArgOperand(0);
4293  Ptr = I.getArgOperand(1);
4294  Mask = I.getArgOperand(2);
4295  Alignment = 0;
4296  };
4297 
4298  Value *PtrOperand, *MaskOperand, *Src0Operand;
4299  unsigned Alignment;
4300  if (IsCompressing)
4301  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4302  else
4303  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4304 
4305  SDValue Ptr = getValue(PtrOperand);
4306  SDValue Src0 = getValue(Src0Operand);
4307  SDValue Mask = getValue(MaskOperand);
4308 
4309  EVT VT = Src0.getValueType();
4310  if (!Alignment)
4311  Alignment = DAG.getEVTAlignment(VT);
4312 
4313  AAMDNodes AAInfo;
4314  I.getAAMetadata(AAInfo);
4315 
4316  MachineMemOperand *MMO =
4317  DAG.getMachineFunction().
4320  Alignment, AAInfo);
4321  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
4322  MMO, false /* Truncating */,
4323  IsCompressing);
4324  DAG.setRoot(StoreNode);
4325  setValue(&I, StoreNode);
4326 }
4327 
4328 // Get a uniform base for the Gather/Scatter intrinsic.
4329 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4330 // We try to represent it as a base pointer + vector of indices.
4331 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4332 // The first operand of the GEP may be a single pointer or a vector of pointers
4333 // Example:
4334 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4335 // or
4336 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4337 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4338 //
4339 // When the first GEP operand is a single pointer - it is the uniform base we
4340 // are looking for. If first operand of the GEP is a splat vector - we
4341 // extract the splat value and use it as a uniform base.
4342 // In all other cases the function returns 'false'.
4343 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
4344  SDValue &Scale, SelectionDAGBuilder* SDB) {
4345  SelectionDAG& DAG = SDB->DAG;
4346  LLVMContext &Context = *DAG.getContext();
4347 
4348  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
4350  if (!GEP)
4351  return false;
4352 
4353  const Value *GEPPtr = GEP->getPointerOperand();
4354  if (!GEPPtr->getType()->isVectorTy())
4355  Ptr = GEPPtr;
4356  else if (!(Ptr = getSplatValue(GEPPtr)))
4357  return false;
4358 
4359  unsigned FinalIndex = GEP->getNumOperands() - 1;
4360  Value *IndexVal = GEP->getOperand(FinalIndex);
4361 
4362  // Ensure all the other indices are 0.
4363  for (unsigned i = 1; i < FinalIndex; ++i) {
4364  auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
4365  if (!C || !C->isZero())
4366  return false;
4367  }
4368 
4369  // The operands of the GEP may be defined in another basic block.
4370  // In this case we'll not find nodes for the operands.
4371  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
4372  return false;
4373 
4374  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4375  const DataLayout &DL = DAG.getDataLayout();
4376  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4377  SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4378  Base = SDB->getValue(Ptr);
4379  Index = SDB->getValue(IndexVal);
4380 
4381  if (!Index.getValueType().isVector()) {
4382  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4383  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4384  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4385  }
4386  return true;
4387 }
4388 
4389 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4390  SDLoc sdl = getCurSDLoc();
4391 
4392  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4393  const Value *Ptr = I.getArgOperand(1);
4394  SDValue Src0 = getValue(I.getArgOperand(0));
4395  SDValue Mask = getValue(I.getArgOperand(3));
4396  EVT VT = Src0.getValueType();
4397  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4398  if (!Alignment)
4399  Alignment = DAG.getEVTAlignment(VT);
4400  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4401 
4402  AAMDNodes AAInfo;
4403  I.getAAMetadata(AAInfo);
4404 
4405  SDValue Base;
4406  SDValue Index;
4407  SDValue Scale;
4408  const Value *BasePtr = Ptr;
4409  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4410 
4411  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4414  MachineMemOperand::MOStore, VT.getStoreSize(),
4415  Alignment, AAInfo);
4416  if (!UniformBase) {
4417  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4418  Index = getValue(Ptr);
4419  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4420  }
4421  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4422  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4423  Ops, MMO);
4424  DAG.setRoot(Scatter);
4425  setValue(&I, Scatter);
4426 }
4427 
4428 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4429  SDLoc sdl = getCurSDLoc();
4430 
4431  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4432  unsigned& Alignment) {
4433  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4434  Ptr = I.getArgOperand(0);
4435  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4436  Mask = I.getArgOperand(2);
4437  Src0 = I.getArgOperand(3);
4438  };
4439  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4440  unsigned& Alignment) {
4441  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4442  Ptr = I.getArgOperand(0);
4443  Alignment = 0;
4444  Mask = I.getArgOperand(1);
4445  Src0 = I.getArgOperand(2);
4446  };
4447 
4448  Value *PtrOperand, *MaskOperand, *Src0Operand;
4449  unsigned Alignment;
4450  if (IsExpanding)
4451  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4452  else
4453  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4454 
4455  SDValue Ptr = getValue(PtrOperand);
4456  SDValue Src0 = getValue(Src0Operand);
4457  SDValue Mask = getValue(MaskOperand);
4458 
4459  EVT VT = Src0.getValueType();
4460  if (!Alignment)
4461  Alignment = DAG.getEVTAlignment(VT);
4462 
4463  AAMDNodes AAInfo;
4464  I.getAAMetadata(AAInfo);
4465  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4466 
4467  // Do not serialize masked loads of constant memory with anything.
4468  bool AddToChain =
4469  !AA || !AA->pointsToConstantMemory(MemoryLocation(
4470  PtrOperand,
4473  AAInfo));
4474  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4475 
4476  MachineMemOperand *MMO =
4477  DAG.getMachineFunction().
4480  Alignment, AAInfo, Ranges);
4481 
4482  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4483  ISD::NON_EXTLOAD, IsExpanding);
4484  if (AddToChain)
4485  PendingLoads.push_back(Load.getValue(1));
4486  setValue(&I, Load);
4487 }
4488 
4489 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4490  SDLoc sdl = getCurSDLoc();
4491 
4492  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4493  const Value *Ptr = I.getArgOperand(0);
4494  SDValue Src0 = getValue(I.getArgOperand(3));
4495  SDValue Mask = getValue(I.getArgOperand(2));
4496 
4497  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4498  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4499  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4500  if (!Alignment)
4501  Alignment = DAG.getEVTAlignment(VT);
4502 
4503  AAMDNodes AAInfo;
4504  I.getAAMetadata(AAInfo);
4505  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4506 
4507  SDValue Root = DAG.getRoot();
4508  SDValue Base;
4509  SDValue Index;
4510  SDValue Scale;
4511  const Value *BasePtr = Ptr;
4512  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4513  bool ConstantMemory = false;
4514  if (UniformBase && AA &&
4515  AA->pointsToConstantMemory(
4516  MemoryLocation(BasePtr,
4519  AAInfo))) {
4520  // Do not serialize (non-volatile) loads of constant memory with anything.
4521  Root = DAG.getEntryNode();
4522  ConstantMemory = true;
4523  }
4524 
4525  MachineMemOperand *MMO =
4526  DAG.getMachineFunction().
4527  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4529  Alignment, AAInfo, Ranges);
4530 
4531  if (!UniformBase) {
4532  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4533  Index = getValue(Ptr);
4534  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4535  }
4536  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4537  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4538  Ops, MMO);
4539 
4540  SDValue OutChain = Gather.getValue(1);
4541  if (!ConstantMemory)
4542  PendingLoads.push_back(OutChain);
4543  setValue(&I, Gather);
4544 }
4545 
4546 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4547  SDLoc dl = getCurSDLoc();
4548  AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4549  AtomicOrdering FailureOrdering = I.getFailureOrdering();
4550  SyncScope::ID SSID = I.getSyncScopeID();
4551 
4552  SDValue InChain = getRoot();
4553 
4554  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4555  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4556 
4557  auto Alignment = DAG.getEVTAlignment(MemVT);
4558 
4560  if (I.isVolatile())
4562  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4563 
4564  MachineFunction &MF = DAG.getMachineFunction();
4565  MachineMemOperand *MMO =
4567  Flags, MemVT.getStoreSize(), Alignment,
4568  AAMDNodes(), nullptr, SSID, SuccessOrdering,
4569  FailureOrdering);
4570 
4572  dl, MemVT, VTs, InChain,
4573  getValue(I.getPointerOperand()),
4574  getValue(I.getCompareOperand()),
4575  getValue(I.getNewValOperand()), MMO);
4576 
4577  SDValue OutChain = L.getValue(2);
4578 
4579  setValue(&I, L);
4580  DAG.setRoot(OutChain);
4581 }
4582 
4583 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4584  SDLoc dl = getCurSDLoc();
4585  ISD::NodeType NT;
4586  switch (I.getOperation()) {
4587  default: llvm_unreachable("Unknown atomicrmw operation");
4588  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4589  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4590  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4591  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4592  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4593  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4594  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4595  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4596  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4597  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4598  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4599  case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4600  case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4601  }
4602  AtomicOrdering Ordering = I.getOrdering();
4603  SyncScope::ID SSID = I.getSyncScopeID();
4604 
4605  SDValue InChain = getRoot();
4606 
4607  auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4608  auto Alignment = DAG.getEVTAlignment(MemVT);
4609 
4611  if (I.isVolatile())
4613  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4614 
4615  MachineFunction &MF = DAG.getMachineFunction();
4616  MachineMemOperand *MMO =
4618  MemVT.getStoreSize(), Alignment, AAMDNodes(),
4619  nullptr, SSID, Ordering);
4620 
4621  SDValue L =
4622  DAG.getAtomic(NT, dl, MemVT, InChain,
4623  getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4624  MMO);
4625 
4626  SDValue OutChain = L.getValue(1);
4627 
4628  setValue(&I, L);
4629  DAG.setRoot(OutChain);
4630 }
4631 
4632 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4633  SDLoc dl = getCurSDLoc();
4634  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4635  SDValue Ops[3];
4636  Ops[0] = getRoot();
4637  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4638  TLI.getFenceOperandTy(DAG.getDataLayout()));
4639  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4640  TLI.getFenceOperandTy(DAG.getDataLayout()));
4641  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4642 }
4643 
4644 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4645  SDLoc dl = getCurSDLoc();
4646  AtomicOrdering Order = I.getOrdering();
4647  SyncScope::ID SSID = I.getSyncScopeID();
4648 
4649  SDValue InChain = getRoot();
4650 
4651  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4652  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4653  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4654 
4655  if (!TLI.supportsUnalignedAtomics() &&
4656  I.getAlignment() < MemVT.getSizeInBits() / 8)
4657  report_fatal_error("Cannot generate unaligned atomic load");
4658 
4659  auto Flags = MachineMemOperand::MOLoad;
4660  if (I.isVolatile())
4662  if (I.getMetadata(LLVMContext::MD_invariant_load) != nullptr)
4666 
4667  Flags |= TLI.getMMOFlags(I);
4668 
4669  MachineMemOperand *MMO =
4670  DAG.getMachineFunction().
4672  Flags, MemVT.getStoreSize(),
4673  I.getAlignment() ? I.getAlignment() :
4674  DAG.getEVTAlignment(MemVT),
4675  AAMDNodes(), nullptr, SSID, Order);
4676 
4677  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4678  SDValue L =
4679  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4680  getValue(I.getPointerOperand()), MMO);
4681 
4682  SDValue OutChain = L.getValue(1);
4683  if (MemVT != VT)
4684  L = DAG.getPtrExtOrTrunc(L, dl, VT);
4685 
4686  setValue(&I, L);
4687  DAG.setRoot(OutChain);
4688 }
4689 
4690 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4691  SDLoc dl = getCurSDLoc();
4692 
4693  AtomicOrdering Ordering = I.getOrdering();
4694  SyncScope::ID SSID = I.getSyncScopeID();
4695 
4696  SDValue InChain = getRoot();
4697 
4698  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4699  EVT MemVT =
4701 
4702  if (I.getAlignment() < MemVT.getSizeInBits() / 8)
4703  report_fatal_error("Cannot generate unaligned atomic store");
4704 
4705  auto Flags = MachineMemOperand::MOStore;
4706  if (I.isVolatile())
4708  Flags |= TLI.getMMOFlags(I);
4709 
4710  MachineFunction &MF = DAG.getMachineFunction();
4711  MachineMemOperand *MMO =
4713  MemVT.getStoreSize(), I.getAlignment(), AAMDNodes(),
4714  nullptr, SSID, Ordering);
4715 
4716  SDValue Val = getValue(I.getValueOperand());
4717  if (Val.getValueType() != MemVT)
4718  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4719 
4720  SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4721  getValue(I.getPointerOperand()), Val, MMO);
4722 
4723 
4724  DAG.setRoot(OutChain);
4725 }
4726 
4727 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4728 /// node.
4729 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4730  unsigned Intrinsic) {
4731  // Ignore the callsite's attributes. A specific call site may be marked with
4732  // readnone, but the lowering code will expect the chain based on the
4733  // definition.
4734  const Function *F = I.getCalledFunction();
4735  bool HasChain = !F->doesNotAccessMemory();
4736  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4737 
4738  // Build the operand list.
4740  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4741  if (OnlyLoad) {
4742  // We don't need to serialize loads against other loads.
4743  Ops.push_back(DAG.getRoot());
4744  } else {
4745  Ops.push_back(getRoot());
4746  }
4747  }
4748 
4749  // Info is set by getTgtMemInstrinsic
4750  TargetLowering::IntrinsicInfo Info;
4751  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4752  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4753  DAG.getMachineFunction(),
4754  Intrinsic);
4755 
4756  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4757  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4758  Info.opc == ISD::INTRINSIC_W_CHAIN)
4759  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4760  TLI.getPointerTy(DAG.getDataLayout())));
4761 
4762  // Add all operands of the call to the operand list.
4763  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4764  SDValue Op = getValue(I.getArgOperand(i));
4765  Ops.push_back(Op);
4766  }
4767 
4769  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4770 
4771  if (HasChain)
4772  ValueVTs.push_back(MVT::Other);
4773 
4774  SDVTList VTs = DAG.getVTList(ValueVTs);
4775 
4776  // Create the node.
4777  SDValue Result;
4778  if (IsTgtIntrinsic) {
4779  // This is target intrinsic that touches memory
4780  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4781  Ops, Info.memVT,
4782  MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4783  Info.flags, Info.size);
4784  } else if (!HasChain) {
4785  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4786  } else if (!I.getType()->isVoidTy()) {
4787  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4788  } else {
4789  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4790  }
4791 
4792  if (HasChain) {
4793  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4794  if (OnlyLoad)
4795  PendingLoads.push_back(Chain);
4796  else
4797  DAG.setRoot(Chain);
4798  }
4799 
4800  if (!I.getType()->isVoidTy()) {
4801  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4802  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4803  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4804  } else
4805  Result = lowerRangeToAssertZExt(DAG, I, Result);
4806 
4807  setValue(&I, Result);
4808  }
4809 }
4810 
4811 /// GetSignificand - Get the significand and build it into a floating-point
4812 /// number with exponent of 1:
4813 ///
4814 /// Op = (Op & 0x007fffff) | 0x3f800000;
4815 ///
4816 /// where Op is the hexadecimal representation of floating point value.
4818  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4819  DAG.getConstant(0x007fffff, dl, MVT::i32));
4820  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4821  DAG.getConstant(0x3f800000, dl, MVT::i32));
4822  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4823 }
4824 
4825 /// GetExponent - Get the exponent:
4826 ///
4827 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4828 ///
4829 /// where Op is the hexadecimal representation of floating point value.
4831  const TargetLowering &TLI, const SDLoc &dl) {
4832  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4833  DAG.getConstant(0x7f800000, dl, MVT::i32));
4834  SDValue t1 = DAG.getNode(
4835  ISD::SRL, dl, MVT::i32, t0,
4836  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4837  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4838  DAG.getConstant(127, dl, MVT::i32));
4839  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4840 }
4841 
4842 /// getF32Constant - Get 32-bit floating point constant.
4843 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4844  const SDLoc &dl) {
4845  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4846  MVT::f32);
4847 }
4848 
4850  SelectionDAG &DAG) {
4851  // TODO: What fast-math-flags should be set on the floating-point nodes?
4852 
4853  // IntegerPartOfX = ((int32_t)(t0);
4854  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4855 
4856  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4857  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4858  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4859 
4860  // IntegerPartOfX <<= 23;
4861  IntegerPartOfX = DAG.getNode(
4862  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4864  DAG.getDataLayout())));
4865 
4866  SDValue TwoToFractionalPartOfX;
4867  if (LimitFloatPrecision <= 6) {
4868  // For floating-point precision of 6:
4869  //
4870  // TwoToFractionalPartOfX =
4871  // 0.997535578f +
4872  // (0.735607626f + 0.252464424f * x) * x;
4873  //
4874  // error 0.0144103317, which is 6 bits
4875  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4876  getF32Constant(DAG, 0x3e814304, dl));
4877  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4878  getF32Constant(DAG, 0x3f3c50c8, dl));
4879  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4880  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4881  getF32Constant(DAG, 0x3f7f5e7e, dl));
4882  } else if (LimitFloatPrecision <= 12) {
4883  // For floating-point precision of 12:
4884  //
4885  // TwoToFractionalPartOfX =
4886  // 0.999892986f +
4887  // (0.696457318f +
4888  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4889  //
4890  // error 0.000107046256, which is 13 to 14 bits
4891  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4892  getF32Constant(DAG, 0x3da235e3, dl));
4893  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4894  getF32Constant(DAG, 0x3e65b8f3, dl));
4895  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4896  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4897  getF32Constant(DAG, 0x3f324b07, dl));
4898  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4899  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4900  getF32Constant(DAG, 0x3f7ff8fd, dl));
4901  } else { // LimitFloatPrecision <= 18
4902  // For floating-point precision of 18:
4903  //
4904  // TwoToFractionalPartOfX =
4905  // 0.999999982f +
4906  // (0.693148872f +
4907  // (0.240227044f +
4908  // (0.554906021e-1f +
4909  // (0.961591928e-2f +
4910  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4911  // error 2.47208000*10^(-7), which is better than 18 bits
4912  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4913  getF32Constant(DAG, 0x3924b03e, dl));
4914  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4915  getF32Constant(DAG, 0x3ab24b87, dl));
4916  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4917  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4918  getF32Constant(DAG, 0x3c1d8c17, dl));
4919  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4920  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4921  getF32Constant(DAG, 0x3d634a1d, dl));
4922  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4923  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4924  getF32Constant(DAG, 0x3e75fe14, dl));
4925  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4926  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4927  getF32Constant(DAG, 0x3f317234, dl));
4928  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4929  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4930  getF32Constant(DAG, 0x3f800000, dl));
4931  }
4932 
4933  // Add the exponent into the result in integer domain.
4934  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4935  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4936  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4937 }
4938 
4939 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4940 /// limited-precision mode.
4941 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4942  const TargetLowering &TLI) {
4943  if (Op.getValueType() == MVT::f32 &&
4944  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4945 
4946  // Put the exponent in the right bit position for later addition to the
4947  // final result:
4948  //
4949  // #define LOG2OFe 1.4426950f
4950  // t0 = Op * LOG2OFe
4951 
4952  // TODO: What fast-math-flags should be set here?
4953  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4954  getF32Constant(DAG, 0x3fb8aa3b, dl));
4955  return getLimitedPrecisionExp2(t0, dl, DAG);
4956  }
4957 
4958  // No special expansion.
4959  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4960 }
4961 
4962 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4963 /// limited-precision mode.
4964 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4965  const TargetLowering &TLI) {
4966  // TODO: What fast-math-flags should be set on the floating-point nodes?
4967 
4968  if (Op.getValueType() == MVT::f32 &&
4969  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4970  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4971 
4972  // Scale the exponent by log(2) [0.69314718f].
4973  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4974  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4975  getF32Constant(DAG, 0x3f317218, dl));
4976 
4977  // Get the significand and build it into a floating-point number with
4978  // exponent of 1.
4979  SDValue X = GetSignificand(DAG, Op1, dl);
4980 
4981  SDValue LogOfMantissa;
4982  if (LimitFloatPrecision <= 6) {
4983  // For floating-point precision of 6:
4984  //
4985  // LogofMantissa =
4986  // -1.1609546f +
4987  // (1.4034025f - 0.23903021f * x) * x;
4988  //
4989  // error 0.0034276066, which is better than 8 bits
4990  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4991  getF32Constant(DAG, 0xbe74c456, dl));
4992  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4993  getF32Constant(DAG, 0x3fb3a2b1, dl));
4994  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4995  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4996  getF32Constant(DAG, 0x3f949a29, dl));
4997  } else if (LimitFloatPrecision <= 12) {
4998  // For floating-point precision of 12:
4999  //
5000  // LogOfMantissa =
5001  // -1.7417939f +
5002  // (2.8212026f +
5003  // (-1.4699568f +
5004  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5005  //
5006  // error 0.000061011436, which is 14 bits
5007  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5008  getF32Constant(DAG, 0xbd67b6d6, dl));
5009  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5010  getF32Constant(DAG, 0x3ee4f4b8, dl));
5011  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5012  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5013  getF32Constant(DAG, 0x3fbc278b, dl));
5014  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5015  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5016  getF32Constant(DAG, 0x40348e95, dl));
5017  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5018  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5019  getF32Constant(DAG, 0x3fdef31a, dl));
5020  } else { // LimitFloatPrecision <= 18
5021  // For floating-point precision of 18:
5022  //
5023  // LogOfMantissa =
5024  // -2.1072184f +
5025  // (4.2372794f +
5026  // (-3.7029485f +
5027  // (2.2781945f +
5028  // (-0.87823314f +
5029  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5030  //
5031  // error 0.0000023660568, which is better than 18 bits
5032  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5033  getF32Constant(DAG, 0xbc91e5ac, dl));
5034  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5035  getF32Constant(DAG, 0x3e4350aa, dl));
5036  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5037  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5038  getF32Constant(DAG, 0x3f60d3e3, dl));
5039  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5040  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5041  getF32Constant(DAG, 0x4011cdf0, dl));
5042  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5043  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5044  getF32Constant(DAG, 0x406cfd1c, dl));
5045  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5046  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5047  getF32Constant(DAG, 0x408797cb, dl));
5048  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5049  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5050  getF32Constant(DAG, 0x4006dcab, dl));
5051  }
5052 
5053  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5054  }
5055 
5056  // No special expansion.
5057  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
5058 }
5059 
5060 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5061 /// limited-precision mode.
5062 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5063  const TargetLowering &TLI) {
5064  // TODO: What fast-math-flags should be set on the floating-point nodes?
5065 
5066  if (Op.getValueType() == MVT::f32 &&
5067  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5068  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5069 
5070  // Get the exponent.
5071  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5072 
5073  // Get the significand and build it into a floating-point number with
5074