LLVM  10.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/BitVector.h"
19 #include "llvm/ADT/DenseMap.h"
20 #include "llvm/ADT/None.h"
21 #include "llvm/ADT/Optional.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/ADT/Twine.h"
33 #include "llvm/Analysis/Loads.h"
38 #include "llvm/CodeGen/Analysis.h"
56 #include "llvm/CodeGen/StackMaps.h"
66 #include "llvm/IR/Argument.h"
67 #include "llvm/IR/Attributes.h"
68 #include "llvm/IR/BasicBlock.h"
69 #include "llvm/IR/CFG.h"
70 #include "llvm/IR/CallSite.h"
71 #include "llvm/IR/CallingConv.h"
72 #include "llvm/IR/Constant.h"
73 #include "llvm/IR/ConstantRange.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/InlineAsm.h"
82 #include "llvm/IR/InstrTypes.h"
83 #include "llvm/IR/Instruction.h"
84 #include "llvm/IR/Instructions.h"
85 #include "llvm/IR/IntrinsicInst.h"
86 #include "llvm/IR/Intrinsics.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Metadata.h"
89 #include "llvm/IR/Module.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Statepoint.h"
93 #include "llvm/IR/Type.h"
94 #include "llvm/IR/User.h"
95 #include "llvm/IR/Value.h"
96 #include "llvm/MC/MCContext.h"
97 #include "llvm/MC/MCSymbol.h"
100 #include "llvm/Support/Casting.h"
101 #include "llvm/Support/CodeGen.h"
103 #include "llvm/Support/Compiler.h"
104 #include "llvm/Support/Debug.h"
107 #include "llvm/Support/MathExtras.h"
113 #include <algorithm>
114 #include <cassert>
115 #include <cstddef>
116 #include <cstdint>
117 #include <cstring>
118 #include <iterator>
119 #include <limits>
120 #include <numeric>
121 #include <tuple>
122 #include <utility>
123 #include <vector>
124 
125 using namespace llvm;
126 using namespace PatternMatch;
127 using namespace SwitchCG;
128 
129 #define DEBUG_TYPE "isel"
130 
131 /// LimitFloatPrecision - Generate low-precision inline sequences for
132 /// some float libcalls (6, 8 or 12 bits).
133 static unsigned LimitFloatPrecision;
134 
136  LimitFPPrecision("limit-float-precision",
137  cl::desc("Generate low-precision inline sequences "
138  "for some float libcalls"),
139  cl::location(LimitFloatPrecision), cl::Hidden,
140  cl::init(0));
141 
143  "switch-peel-threshold", cl::Hidden, cl::init(66),
144  cl::desc("Set the case probability threshold for peeling the case from a "
145  "switch statement. A value greater than 100 will void this "
146  "optimization"));
147 
148 // Limit the width of DAG chains. This is important in general to prevent
149 // DAG-based analysis from blowing up. For example, alias analysis and
150 // load clustering may not complete in reasonable time. It is difficult to
151 // recognize and avoid this situation within each individual analysis, and
152 // future analyses are likely to have the same behavior. Limiting DAG width is
153 // the safe approach and will be especially important with global DAGs.
154 //
155 // MaxParallelChains default is arbitrarily high to avoid affecting
156 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
157 // sequence over this should have been converted to llvm.memcpy by the
158 // frontend. It is easy to induce this behavior with .ll code such as:
159 // %buffer = alloca [4096 x i8]
160 // %data = load [4096 x i8]* %argPtr
161 // store [4096 x i8] %data, [4096 x i8]* %buffer
162 static const unsigned MaxParallelChains = 64;
163 
164 // Return the calling convention if the Value passed requires ABI mangling as it
165 // is a parameter to a function or a return value from a function which is not
166 // an intrinsic.
168  if (auto *R = dyn_cast<ReturnInst>(V))
169  return R->getParent()->getParent()->getCallingConv();
170 
171  if (auto *CI = dyn_cast<CallInst>(V)) {
172  const bool IsInlineAsm = CI->isInlineAsm();
173  const bool IsIndirectFunctionCall =
174  !IsInlineAsm && !CI->getCalledFunction();
175 
176  // It is possible that the call instruction is an inline asm statement or an
177  // indirect function call in which case the return value of
178  // getCalledFunction() would be nullptr.
179  const bool IsInstrinsicCall =
180  !IsInlineAsm && !IsIndirectFunctionCall &&
181  CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
182 
183  if (!IsInlineAsm && !IsInstrinsicCall)
184  return CI->getCallingConv();
185  }
186 
187  return None;
188 }
189 
190 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
191  const SDValue *Parts, unsigned NumParts,
192  MVT PartVT, EVT ValueVT, const Value *V,
194 
195 /// getCopyFromParts - Create a value that contains the specified legal parts
196 /// combined into the value they represent. If the parts combine to a type
197 /// larger than ValueVT then AssertOp can be used to specify whether the extra
198 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
199 /// (ISD::AssertSext).
201  const SDValue *Parts, unsigned NumParts,
202  MVT PartVT, EVT ValueVT, const Value *V,
203  Optional<CallingConv::ID> CC = None,
204  Optional<ISD::NodeType> AssertOp = None) {
205  if (ValueVT.isVector())
206  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
207  CC);
208 
209  assert(NumParts > 0 && "No parts to assemble!");
210  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
211  SDValue Val = Parts[0];
212 
213  if (NumParts > 1) {
214  // Assemble the value from multiple parts.
215  if (ValueVT.isInteger()) {
216  unsigned PartBits = PartVT.getSizeInBits();
217  unsigned ValueBits = ValueVT.getSizeInBits();
218 
219  // Assemble the power of 2 part.
220  unsigned RoundParts =
221  (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
222  unsigned RoundBits = PartBits * RoundParts;
223  EVT RoundVT = RoundBits == ValueBits ?
224  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
225  SDValue Lo, Hi;
226 
227  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
228 
229  if (RoundParts > 2) {
230  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
231  PartVT, HalfVT, V);
232  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
233  RoundParts / 2, PartVT, HalfVT, V);
234  } else {
235  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
236  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
237  }
238 
239  if (DAG.getDataLayout().isBigEndian())
240  std::swap(Lo, Hi);
241 
242  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
243 
244  if (RoundParts < NumParts) {
245  // Assemble the trailing non-power-of-2 part.
246  unsigned OddParts = NumParts - RoundParts;
247  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
248  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
249  OddVT, V, CC);
250 
251  // Combine the round and odd parts.
252  Lo = Val;
253  if (DAG.getDataLayout().isBigEndian())
254  std::swap(Lo, Hi);
255  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
256  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
257  Hi =
258  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
259  DAG.getConstant(Lo.getValueSizeInBits(), DL,
260  TLI.getPointerTy(DAG.getDataLayout())));
261  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
262  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
263  }
264  } else if (PartVT.isFloatingPoint()) {
265  // FP split into multiple FP parts (for ppcf128)
266  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
267  "Unexpected split");
268  SDValue Lo, Hi;
269  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
270  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
271  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
272  std::swap(Lo, Hi);
273  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
274  } else {
275  // FP split into integer parts (soft fp)
276  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
277  !PartVT.isVector() && "Unexpected split");
278  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
279  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
280  }
281  }
282 
283  // There is now one part, held in Val. Correct it to match ValueVT.
284  // PartEVT is the type of the register class that holds the value.
285  // ValueVT is the type of the inline asm operation.
286  EVT PartEVT = Val.getValueType();
287 
288  if (PartEVT == ValueVT)
289  return Val;
290 
291  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
292  ValueVT.bitsLT(PartEVT)) {
293  // For an FP value in an integer part, we need to truncate to the right
294  // width first.
295  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
296  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
297  }
298 
299  // Handle types that have the same size.
300  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
301  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
302 
303  // Handle types with different sizes.
304  if (PartEVT.isInteger() && ValueVT.isInteger()) {
305  if (ValueVT.bitsLT(PartEVT)) {
306  // For a truncate, see if we have any information to
307  // indicate whether the truncated bits will always be
308  // zero or sign-extension.
309  if (AssertOp.hasValue())
310  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
311  DAG.getValueType(ValueVT));
312  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
313  }
314  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
315  }
316 
317  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
318  // FP_ROUND's are always exact here.
319  if (ValueVT.bitsLT(Val.getValueType()))
320  return DAG.getNode(
321  ISD::FP_ROUND, DL, ValueVT, Val,
322  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
323 
324  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
325  }
326 
327  // Handle MMX to a narrower integer type by bitcasting MMX to integer and
328  // then truncating.
329  if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
330  ValueVT.bitsLT(PartEVT)) {
331  Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
332  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
333  }
334 
335  report_fatal_error("Unknown mismatch in getCopyFromParts!");
336 }
337 
339  const Twine &ErrMsg) {
340  const Instruction *I = dyn_cast_or_null<Instruction>(V);
341  if (!V)
342  return Ctx.emitError(ErrMsg);
343 
344  const char *AsmError = ", possible invalid constraint for vector type";
345  if (const CallInst *CI = dyn_cast<CallInst>(I))
346  if (isa<InlineAsm>(CI->getCalledValue()))
347  return Ctx.emitError(I, ErrMsg + AsmError);
348 
349  return Ctx.emitError(I, ErrMsg);
350 }
351 
352 /// getCopyFromPartsVector - Create a value that contains the specified legal
353 /// parts combined into the value they represent. If the parts combine to a
354 /// type larger than ValueVT then AssertOp can be used to specify whether the
355 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
356 /// ValueVT (ISD::AssertSext).
358  const SDValue *Parts, unsigned NumParts,
359  MVT PartVT, EVT ValueVT, const Value *V,
360  Optional<CallingConv::ID> CallConv) {
361  assert(ValueVT.isVector() && "Not a vector value");
362  assert(NumParts > 0 && "No parts to assemble!");
363  const bool IsABIRegCopy = CallConv.hasValue();
364 
365  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
366  SDValue Val = Parts[0];
367 
368  // Handle a multi-element vector.
369  if (NumParts > 1) {
370  EVT IntermediateVT;
371  MVT RegisterVT;
372  unsigned NumIntermediates;
373  unsigned NumRegs;
374 
375  if (IsABIRegCopy) {
377  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
378  NumIntermediates, RegisterVT);
379  } else {
380  NumRegs =
381  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
382  NumIntermediates, RegisterVT);
383  }
384 
385  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
386  NumParts = NumRegs; // Silence a compiler warning.
387  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
388  assert(RegisterVT.getSizeInBits() ==
389  Parts[0].getSimpleValueType().getSizeInBits() &&
390  "Part type sizes don't match!");
391 
392  // Assemble the parts into intermediate operands.
393  SmallVector<SDValue, 8> Ops(NumIntermediates);
394  if (NumIntermediates == NumParts) {
395  // If the register was not expanded, truncate or copy the value,
396  // as appropriate.
397  for (unsigned i = 0; i != NumParts; ++i)
398  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
399  PartVT, IntermediateVT, V);
400  } else if (NumParts > 0) {
401  // If the intermediate type was expanded, build the intermediate
402  // operands from the parts.
403  assert(NumParts % NumIntermediates == 0 &&
404  "Must expand into a divisible number of parts!");
405  unsigned Factor = NumParts / NumIntermediates;
406  for (unsigned i = 0; i != NumIntermediates; ++i)
407  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
408  PartVT, IntermediateVT, V);
409  }
410 
411  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
412  // intermediate operands.
413  EVT BuiltVectorTy =
414  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
415  (IntermediateVT.isVector()
416  ? IntermediateVT.getVectorNumElements() * NumParts
417  : NumIntermediates));
418  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
420  DL, BuiltVectorTy, Ops);
421  }
422 
423  // There is now one part, held in Val. Correct it to match ValueVT.
424  EVT PartEVT = Val.getValueType();
425 
426  if (PartEVT == ValueVT)
427  return Val;
428 
429  if (PartEVT.isVector()) {
430  // If the element type of the source/dest vectors are the same, but the
431  // parts vector has more elements than the value vector, then we have a
432  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
433  // elements we want.
434  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
435  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
436  "Cannot narrow, it would be a lossy transformation");
437  return DAG.getNode(
438  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
439  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
440  }
441 
442  // Vector/Vector bitcast.
443  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
444  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
445 
446  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
447  "Cannot handle this kind of promotion");
448  // Promoted vector extract
449  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
450 
451  }
452 
453  // Trivial bitcast if the types are the same size and the destination
454  // vector type is legal.
455  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
456  TLI.isTypeLegal(ValueVT))
457  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
458 
459  if (ValueVT.getVectorNumElements() != 1) {
460  // Certain ABIs require that vectors are passed as integers. For vectors
461  // are the same size, this is an obvious bitcast.
462  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
463  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
464  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
465  // Bitcast Val back the original type and extract the corresponding
466  // vector we want.
467  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
468  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
469  ValueVT.getVectorElementType(), Elts);
470  Val = DAG.getBitcast(WiderVecType, Val);
471  return DAG.getNode(
472  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
473  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
474  }
475 
477  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
478  return DAG.getUNDEF(ValueVT);
479  }
480 
481  // Handle cases such as i8 -> <1 x i1>
482  EVT ValueSVT = ValueVT.getVectorElementType();
483  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
484  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
485  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
486 
487  return DAG.getBuildVector(ValueVT, DL, Val);
488 }
489 
490 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
491  SDValue Val, SDValue *Parts, unsigned NumParts,
492  MVT PartVT, const Value *V,
493  Optional<CallingConv::ID> CallConv);
494 
495 /// getCopyToParts - Create a series of nodes that contain the specified value
496 /// split into legal parts. If the parts contain more bits than Val, then, for
497 /// integers, ExtendKind can be used to specify how to generate the extra bits.
498 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
499  SDValue *Parts, unsigned NumParts, MVT PartVT,
500  const Value *V,
501  Optional<CallingConv::ID> CallConv = None,
502  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
503  EVT ValueVT = Val.getValueType();
504 
505  // Handle the vector case separately.
506  if (ValueVT.isVector())
507  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
508  CallConv);
509 
510  unsigned PartBits = PartVT.getSizeInBits();
511  unsigned OrigNumParts = NumParts;
512  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
513  "Copying to an illegal type!");
514 
515  if (NumParts == 0)
516  return;
517 
518  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
519  EVT PartEVT = PartVT;
520  if (PartEVT == ValueVT) {
521  assert(NumParts == 1 && "No-op copy with multiple parts!");
522  Parts[0] = Val;
523  return;
524  }
525 
526  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
527  // If the parts cover more bits than the value has, promote the value.
528  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
529  assert(NumParts == 1 && "Do not know what to promote to!");
530  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
531  } else {
532  if (ValueVT.isFloatingPoint()) {
533  // FP values need to be bitcast, then extended if they are being put
534  // into a larger container.
535  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
536  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
537  }
538  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
539  ValueVT.isInteger() &&
540  "Unknown mismatch!");
541  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
542  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
543  if (PartVT == MVT::x86mmx)
544  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
545  }
546  } else if (PartBits == ValueVT.getSizeInBits()) {
547  // Different types of the same size.
548  assert(NumParts == 1 && PartEVT != ValueVT);
549  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
550  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
551  // If the parts cover less bits than value has, truncate the value.
552  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
553  ValueVT.isInteger() &&
554  "Unknown mismatch!");
555  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
556  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
557  if (PartVT == MVT::x86mmx)
558  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559  }
560 
561  // The value may have changed - recompute ValueVT.
562  ValueVT = Val.getValueType();
563  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
564  "Failed to tile the value with PartVT!");
565 
566  if (NumParts == 1) {
567  if (PartEVT != ValueVT) {
569  "scalar-to-vector conversion failed");
570  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
571  }
572 
573  Parts[0] = Val;
574  return;
575  }
576 
577  // Expand the value into multiple parts.
578  if (NumParts & (NumParts - 1)) {
579  // The number of parts is not a power of 2. Split off and copy the tail.
580  assert(PartVT.isInteger() && ValueVT.isInteger() &&
581  "Do not know what to expand to!");
582  unsigned RoundParts = 1 << Log2_32(NumParts);
583  unsigned RoundBits = RoundParts * PartBits;
584  unsigned OddParts = NumParts - RoundParts;
585  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
586  DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
587 
588  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
589  CallConv);
590 
591  if (DAG.getDataLayout().isBigEndian())
592  // The odd parts were reversed by getCopyToParts - unreverse them.
593  std::reverse(Parts + RoundParts, Parts + NumParts);
594 
595  NumParts = RoundParts;
596  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
597  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
598  }
599 
600  // The number of parts is a power of 2. Repeatedly bisect the value using
601  // EXTRACT_ELEMENT.
602  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
604  ValueVT.getSizeInBits()),
605  Val);
606 
607  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
608  for (unsigned i = 0; i < NumParts; i += StepSize) {
609  unsigned ThisBits = StepSize * PartBits / 2;
610  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
611  SDValue &Part0 = Parts[i];
612  SDValue &Part1 = Parts[i+StepSize/2];
613 
614  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
615  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
616  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
617  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
618 
619  if (ThisBits == PartBits && ThisVT != PartVT) {
620  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
621  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
622  }
623  }
624  }
625 
626  if (DAG.getDataLayout().isBigEndian())
627  std::reverse(Parts, Parts + OrigNumParts);
628 }
629 
631  SDValue Val, const SDLoc &DL, EVT PartVT) {
632  if (!PartVT.isVector())
633  return SDValue();
634 
635  EVT ValueVT = Val.getValueType();
636  unsigned PartNumElts = PartVT.getVectorNumElements();
637  unsigned ValueNumElts = ValueVT.getVectorNumElements();
638  if (PartNumElts > ValueNumElts &&
639  PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
640  EVT ElementVT = PartVT.getVectorElementType();
641  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
642  // undef elements.
644  DAG.ExtractVectorElements(Val, Ops);
645  SDValue EltUndef = DAG.getUNDEF(ElementVT);
646  for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
647  Ops.push_back(EltUndef);
648 
649  // FIXME: Use CONCAT for 2x -> 4x.
650  return DAG.getBuildVector(PartVT, DL, Ops);
651  }
652 
653  return SDValue();
654 }
655 
656 /// getCopyToPartsVector - Create a series of nodes that contain the specified
657 /// value split into legal parts.
658 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
659  SDValue Val, SDValue *Parts, unsigned NumParts,
660  MVT PartVT, const Value *V,
661  Optional<CallingConv::ID> CallConv) {
662  EVT ValueVT = Val.getValueType();
663  assert(ValueVT.isVector() && "Not a vector");
664  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
665  const bool IsABIRegCopy = CallConv.hasValue();
666 
667  if (NumParts == 1) {
668  EVT PartEVT = PartVT;
669  if (PartEVT == ValueVT) {
670  // Nothing to do.
671  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
672  // Bitconvert vector->vector case.
673  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
674  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
675  Val = Widened;
676  } else if (PartVT.isVector() &&
677  PartEVT.getVectorElementType().bitsGE(
678  ValueVT.getVectorElementType()) &&
679  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
680 
681  // Promoted vector extract
682  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
683  } else {
684  if (ValueVT.getVectorNumElements() == 1) {
685  Val = DAG.getNode(
686  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
687  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
688  } else {
689  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
690  "lossy conversion of vector to scalar type");
691  EVT IntermediateType =
692  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
693  Val = DAG.getBitcast(IntermediateType, Val);
694  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
695  }
696  }
697 
698  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
699  Parts[0] = Val;
700  return;
701  }
702 
703  // Handle a multi-element vector.
704  EVT IntermediateVT;
705  MVT RegisterVT;
706  unsigned NumIntermediates;
707  unsigned NumRegs;
708  if (IsABIRegCopy) {
709  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
710  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
711  NumIntermediates, RegisterVT);
712  } else {
713  NumRegs =
714  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
715  NumIntermediates, RegisterVT);
716  }
717 
718  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
719  NumParts = NumRegs; // Silence a compiler warning.
720  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
721 
722  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
723  IntermediateVT.getVectorNumElements() : 1;
724 
725  // Convert the vector to the appropiate type if necessary.
726  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
727 
728  EVT BuiltVectorTy = EVT::getVectorVT(
729  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
730  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
731  if (ValueVT != BuiltVectorTy) {
732  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
733  Val = Widened;
734 
735  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
736  }
737 
738  // Split the vector into intermediate operands.
739  SmallVector<SDValue, 8> Ops(NumIntermediates);
740  for (unsigned i = 0; i != NumIntermediates; ++i) {
741  if (IntermediateVT.isVector()) {
742  Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
743  DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
744  } else {
745  Ops[i] = DAG.getNode(
746  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
747  DAG.getConstant(i, DL, IdxVT));
748  }
749  }
750 
751  // Split the intermediate operands into legal parts.
752  if (NumParts == NumIntermediates) {
753  // If the register was not expanded, promote or copy the value,
754  // as appropriate.
755  for (unsigned i = 0; i != NumParts; ++i)
756  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
757  } else if (NumParts > 0) {
758  // If the intermediate type was expanded, split each the value into
759  // legal parts.
760  assert(NumIntermediates != 0 && "division by zero");
761  assert(NumParts % NumIntermediates == 0 &&
762  "Must expand into a divisible number of parts!");
763  unsigned Factor = NumParts / NumIntermediates;
764  for (unsigned i = 0; i != NumIntermediates; ++i)
765  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
766  CallConv);
767  }
768 }
769 
771  EVT valuevt, Optional<CallingConv::ID> CC)
772  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
773  RegCount(1, regs.size()), CallConv(CC) {}
774 
776  const DataLayout &DL, unsigned Reg, Type *Ty,
778  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
779 
780  CallConv = CC;
781 
782  for (EVT ValueVT : ValueVTs) {
783  unsigned NumRegs =
784  isABIMangled()
785  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
786  : TLI.getNumRegisters(Context, ValueVT);
787  MVT RegisterVT =
788  isABIMangled()
789  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
790  : TLI.getRegisterType(Context, ValueVT);
791  for (unsigned i = 0; i != NumRegs; ++i)
792  Regs.push_back(Reg + i);
793  RegVTs.push_back(RegisterVT);
794  RegCount.push_back(NumRegs);
795  Reg += NumRegs;
796  }
797 }
798 
800  FunctionLoweringInfo &FuncInfo,
801  const SDLoc &dl, SDValue &Chain,
802  SDValue *Flag, const Value *V) const {
803  // A Value with type {} or [0 x %t] needs no registers.
804  if (ValueVTs.empty())
805  return SDValue();
806 
807  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
808 
809  // Assemble the legal parts into the final values.
810  SmallVector<SDValue, 4> Values(ValueVTs.size());
812  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
813  // Copy the legal parts from the registers.
814  EVT ValueVT = ValueVTs[Value];
815  unsigned NumRegs = RegCount[Value];
816  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
817  *DAG.getContext(),
818  CallConv.getValue(), RegVTs[Value])
819  : RegVTs[Value];
820 
821  Parts.resize(NumRegs);
822  for (unsigned i = 0; i != NumRegs; ++i) {
823  SDValue P;
824  if (!Flag) {
825  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
826  } else {
827  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
828  *Flag = P.getValue(2);
829  }
830 
831  Chain = P.getValue(1);
832  Parts[i] = P;
833 
834  // If the source register was virtual and if we know something about it,
835  // add an assert node.
836  if (!Register::isVirtualRegister(Regs[Part + i]) ||
837  !RegisterVT.isInteger())
838  continue;
839 
841  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
842  if (!LOI)
843  continue;
844 
845  unsigned RegSize = RegisterVT.getScalarSizeInBits();
846  unsigned NumSignBits = LOI->NumSignBits;
847  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
848 
849  if (NumZeroBits == RegSize) {
850  // The current value is a zero.
851  // Explicitly express that as it would be easier for
852  // optimizations to kick in.
853  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
854  continue;
855  }
856 
857  // FIXME: We capture more information than the dag can represent. For
858  // now, just use the tightest assertzext/assertsext possible.
859  bool isSExt;
860  EVT FromVT(MVT::Other);
861  if (NumZeroBits) {
862  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
863  isSExt = false;
864  } else if (NumSignBits > 1) {
865  FromVT =
866  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
867  isSExt = true;
868  } else {
869  continue;
870  }
871  // Add an assertion node.
872  assert(FromVT != MVT::Other);
873  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
874  RegisterVT, P, DAG.getValueType(FromVT));
875  }
876 
877  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
878  RegisterVT, ValueVT, V, CallConv);
879  Part += NumRegs;
880  Parts.clear();
881  }
882 
883  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
884 }
885 
887  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
888  const Value *V,
889  ISD::NodeType PreferredExtendType) const {
890  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
891  ISD::NodeType ExtendKind = PreferredExtendType;
892 
893  // Get the list of the values's legal parts.
894  unsigned NumRegs = Regs.size();
895  SmallVector<SDValue, 8> Parts(NumRegs);
896  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
897  unsigned NumParts = RegCount[Value];
898 
899  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
900  *DAG.getContext(),
901  CallConv.getValue(), RegVTs[Value])
902  : RegVTs[Value];
903 
904  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
905  ExtendKind = ISD::ZERO_EXTEND;
906 
907  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
908  NumParts, RegisterVT, V, CallConv, ExtendKind);
909  Part += NumParts;
910  }
911 
912  // Copy the parts into the registers.
913  SmallVector<SDValue, 8> Chains(NumRegs);
914  for (unsigned i = 0; i != NumRegs; ++i) {
915  SDValue Part;
916  if (!Flag) {
917  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
918  } else {
919  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
920  *Flag = Part.getValue(1);
921  }
922 
923  Chains[i] = Part.getValue(0);
924  }
925 
926  if (NumRegs == 1 || Flag)
927  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
928  // flagged to it. That is the CopyToReg nodes and the user are considered
929  // a single scheduling unit. If we create a TokenFactor and return it as
930  // chain, then the TokenFactor is both a predecessor (operand) of the
931  // user as well as a successor (the TF operands are flagged to the user).
932  // c1, f1 = CopyToReg
933  // c2, f2 = CopyToReg
934  // c3 = TokenFactor c1, c2
935  // ...
936  // = op c3, ..., f2
937  Chain = Chains[NumRegs-1];
938  else
939  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
940 }
941 
942 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
943  unsigned MatchingIdx, const SDLoc &dl,
944  SelectionDAG &DAG,
945  std::vector<SDValue> &Ops) const {
946  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
947 
948  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
949  if (HasMatching)
950  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
951  else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
952  // Put the register class of the virtual registers in the flag word. That
953  // way, later passes can recompute register class constraints for inline
954  // assembly as well as normal instructions.
955  // Don't do this for tied operands that can use the regclass information
956  // from the def.
958  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
959  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
960  }
961 
962  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
963  Ops.push_back(Res);
964 
965  if (Code == InlineAsm::Kind_Clobber) {
966  // Clobbers should always have a 1:1 mapping with registers, and may
967  // reference registers that have illegal (e.g. vector) types. Hence, we
968  // shouldn't try to apply any sort of splitting logic to them.
969  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
970  "No 1:1 mapping from clobbers to regs?");
971  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
972  (void)SP;
973  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
974  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
975  assert(
976  (Regs[I] != SP ||
978  "If we clobbered the stack pointer, MFI should know about it.");
979  }
980  return;
981  }
982 
983  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
984  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
985  MVT RegisterVT = RegVTs[Value];
986  for (unsigned i = 0; i != NumRegs; ++i) {
987  assert(Reg < Regs.size() && "Mismatch in # registers expected");
988  unsigned TheReg = Regs[Reg++];
989  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
990  }
991  }
992 }
993 
997  unsigned I = 0;
998  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
999  unsigned RegCount = std::get<0>(CountAndVT);
1000  MVT RegisterVT = std::get<1>(CountAndVT);
1001  unsigned RegisterSize = RegisterVT.getSizeInBits();
1002  for (unsigned E = I + RegCount; I != E; ++I)
1003  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1004  }
1005  return OutVec;
1006 }
1007 
1009  const TargetLibraryInfo *li) {
1010  AA = aa;
1011  GFI = gfi;
1012  LibInfo = li;
1013  DL = &DAG.getDataLayout();
1014  Context = DAG.getContext();
1015  LPadToCallSiteMap.clear();
1016  SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1017 }
1018 
1020  NodeMap.clear();
1021  UnusedArgNodeMap.clear();
1022  PendingLoads.clear();
1023  PendingExports.clear();
1024  CurInst = nullptr;
1025  HasTailCall = false;
1026  SDNodeOrder = LowestSDNodeOrder;
1027  StatepointLowering.clear();
1028 }
1029 
1031  DanglingDebugInfoMap.clear();
1032 }
1033 
1035  if (PendingLoads.empty())
1036  return DAG.getRoot();
1037 
1038  if (PendingLoads.size() == 1) {
1039  SDValue Root = PendingLoads[0];
1040  DAG.setRoot(Root);
1041  PendingLoads.clear();
1042  return Root;
1043  }
1044 
1045  // Otherwise, we have to make a token factor node.
1046  SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1047  PendingLoads.clear();
1048  DAG.setRoot(Root);
1049  return Root;
1050 }
1051 
1053  SDValue Root = DAG.getRoot();
1054 
1055  if (PendingExports.empty())
1056  return Root;
1057 
1058  // Turn all of the CopyToReg chains into one factored node.
1059  if (Root.getOpcode() != ISD::EntryToken) {
1060  unsigned i = 0, e = PendingExports.size();
1061  for (; i != e; ++i) {
1062  assert(PendingExports[i].getNode()->getNumOperands() > 1);
1063  if (PendingExports[i].getNode()->getOperand(0) == Root)
1064  break; // Don't add the root if we already indirectly depend on it.
1065  }
1066 
1067  if (i == e)
1068  PendingExports.push_back(Root);
1069  }
1070 
1071  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1072  PendingExports);
1073  PendingExports.clear();
1074  DAG.setRoot(Root);
1075  return Root;
1076 }
1077 
1079  // Set up outgoing PHI node register values before emitting the terminator.
1080  if (I.isTerminator()) {
1081  HandlePHINodesInSuccessorBlocks(I.getParent());
1082  }
1083 
1084  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1085  if (!isa<DbgInfoIntrinsic>(I))
1086  ++SDNodeOrder;
1087 
1088  CurInst = &I;
1089 
1090  visit(I.getOpcode(), I);
1091 
1092  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1093  // Propagate the fast-math-flags of this IR instruction to the DAG node that
1094  // maps to this instruction.
1095  // TODO: We could handle all flags (nsw, etc) here.
1096  // TODO: If an IR instruction maps to >1 node, only the final node will have
1097  // flags set.
1098  if (SDNode *Node = getNodeForIRValue(&I)) {
1099  SDNodeFlags IncomingFlags;
1100  IncomingFlags.copyFMF(*FPMO);
1101  if (!Node->getFlags().isDefined())
1102  Node->setFlags(IncomingFlags);
1103  else
1104  Node->intersectFlagsWith(IncomingFlags);
1105  }
1106  }
1107 
1108  if (!I.isTerminator() && !HasTailCall &&
1109  !isStatepoint(&I)) // statepoints handle their exports internally
1110  CopyToExportRegsIfNeeded(&I);
1111 
1112  CurInst = nullptr;
1113 }
1114 
1115 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1116  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1117 }
1118 
1119 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1120  // Note: this doesn't use InstVisitor, because it has to work with
1121  // ConstantExpr's in addition to instructions.
1122  switch (Opcode) {
1123  default: llvm_unreachable("Unknown instruction type encountered!");
1124  // Build the switch statement using the Instruction.def file.
1125 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1126  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1127 #include "llvm/IR/Instruction.def"
1128  }
1129 }
1130 
1132  const DIExpression *Expr) {
1133  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1134  const DbgValueInst *DI = DDI.getDI();
1135  DIVariable *DanglingVariable = DI->getVariable();
1136  DIExpression *DanglingExpr = DI->getExpression();
1137  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1138  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1139  return true;
1140  }
1141  return false;
1142  };
1143 
1144  for (auto &DDIMI : DanglingDebugInfoMap) {
1145  DanglingDebugInfoVector &DDIV = DDIMI.second;
1146 
1147  // If debug info is to be dropped, run it through final checks to see
1148  // whether it can be salvaged.
1149  for (auto &DDI : DDIV)
1150  if (isMatchingDbgValue(DDI))
1151  salvageUnresolvedDbgValue(DDI);
1152 
1153  DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1154  }
1155 }
1156 
1157 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1158 // generate the debug data structures now that we've seen its definition.
1160  SDValue Val) {
1161  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1162  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1163  return;
1164 
1165  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1166  for (auto &DDI : DDIV) {
1167  const DbgValueInst *DI = DDI.getDI();
1168  assert(DI && "Ill-formed DanglingDebugInfo");
1169  DebugLoc dl = DDI.getdl();
1170  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1171  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1172  DILocalVariable *Variable = DI->getVariable();
1173  DIExpression *Expr = DI->getExpression();
1174  assert(Variable->isValidLocationForIntrinsic(dl) &&
1175  "Expected inlined-at fields to agree");
1176  SDDbgValue *SDV;
1177  if (Val.getNode()) {
1178  // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1179  // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1180  // we couldn't resolve it directly when examining the DbgValue intrinsic
1181  // in the first place we should not be more successful here). Unless we
1182  // have some test case that prove this to be correct we should avoid
1183  // calling EmitFuncArgumentDbgValue here.
1184  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1185  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1186  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1187  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1188  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1189  // inserted after the definition of Val when emitting the instructions
1190  // after ISel. An alternative could be to teach
1191  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1192  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1193  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1194  << ValSDNodeOrder << "\n");
1195  SDV = getDbgValue(Val, Variable, Expr, dl,
1196  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1197  DAG.AddDbgValue(SDV, Val.getNode(), false);
1198  } else
1199  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1200  << "in EmitFuncArgumentDbgValue\n");
1201  } else {
1202  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1203  auto Undef =
1204  UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1205  auto SDV =
1206  DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1207  DAG.AddDbgValue(SDV, nullptr, false);
1208  }
1209  }
1210  DDIV.clear();
1211 }
1212 
1214  Value *V = DDI.getDI()->getValue();
1215  DILocalVariable *Var = DDI.getDI()->getVariable();
1216  DIExpression *Expr = DDI.getDI()->getExpression();
1217  DebugLoc DL = DDI.getdl();
1218  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1219  unsigned SDOrder = DDI.getSDNodeOrder();
1220 
1221  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1222  // that DW_OP_stack_value is desired.
1223  assert(isa<DbgValueInst>(DDI.getDI()));
1224  bool StackValue = true;
1225 
1226  // Can this Value can be encoded without any further work?
1227  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1228  return;
1229 
1230  // Attempt to salvage back through as many instructions as possible. Bail if
1231  // a non-instruction is seen, such as a constant expression or global
1232  // variable. FIXME: Further work could recover those too.
1233  while (isa<Instruction>(V)) {
1234  Instruction &VAsInst = *cast<Instruction>(V);
1235  DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
1236 
1237  // If we cannot salvage any further, and haven't yet found a suitable debug
1238  // expression, bail out.
1239  if (!NewExpr)
1240  break;
1241 
1242  // New value and expr now represent this debuginfo.
1243  V = VAsInst.getOperand(0);
1244  Expr = NewExpr;
1245 
1246  // Some kind of simplification occurred: check whether the operand of the
1247  // salvaged debug expression can be encoded in this DAG.
1248  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1249  LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "
1250  << DDI.getDI() << "\nBy stripping back to:\n " << V);
1251  return;
1252  }
1253  }
1254 
1255  // This was the final opportunity to salvage this debug information, and it
1256  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1257  // any earlier variable location.
1258  auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1259  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1260  DAG.AddDbgValue(SDV, nullptr, false);
1261 
1262  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()
1263  << "\n");
1264  LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)
1265  << "\n");
1266 }
1267 
1269  DIExpression *Expr, DebugLoc dl,
1270  DebugLoc InstDL, unsigned Order) {
1271  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1272  SDDbgValue *SDV;
1273  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1274  isa<ConstantPointerNull>(V)) {
1275  SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
1276  DAG.AddDbgValue(SDV, nullptr, false);
1277  return true;
1278  }
1279 
1280  // If the Value is a frame index, we can create a FrameIndex debug value
1281  // without relying on the DAG at all.
1282  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1283  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1284  if (SI != FuncInfo.StaticAllocaMap.end()) {
1285  auto SDV =
1286  DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
1287  /*IsIndirect*/ false, dl, SDNodeOrder);
1288  // Do not attach the SDNodeDbgValue to an SDNode: this variable location
1289  // is still available even if the SDNode gets optimized out.
1290  DAG.AddDbgValue(SDV, nullptr, false);
1291  return true;
1292  }
1293  }
1294 
1295  // Do not use getValue() in here; we don't want to generate code at
1296  // this point if it hasn't been done yet.
1297  SDValue N = NodeMap[V];
1298  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1299  N = UnusedArgNodeMap[V];
1300  if (N.getNode()) {
1301  if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1302  return true;
1303  SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1304  DAG.AddDbgValue(SDV, N.getNode(), false);
1305  return true;
1306  }
1307 
1308  // Special rules apply for the first dbg.values of parameter variables in a
1309  // function. Identify them by the fact they reference Argument Values, that
1310  // they're parameters, and they are parameters of the current function. We
1311  // need to let them dangle until they get an SDNode.
1312  bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&
1313  !InstDL.getInlinedAt();
1314  if (!IsParamOfFunc) {
1315  // The value is not used in this block yet (or it would have an SDNode).
1316  // We still want the value to appear for the user if possible -- if it has
1317  // an associated VReg, we can refer to that instead.
1318  auto VMI = FuncInfo.ValueMap.find(V);
1319  if (VMI != FuncInfo.ValueMap.end()) {
1320  unsigned Reg = VMI->second;
1321  // If this is a PHI node, it may be split up into several MI PHI nodes
1322  // (in FunctionLoweringInfo::set).
1323  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1324  V->getType(), None);
1325  if (RFV.occupiesMultipleRegs()) {
1326  unsigned Offset = 0;
1327  unsigned BitsToDescribe = 0;
1328  if (auto VarSize = Var->getSizeInBits())
1329  BitsToDescribe = *VarSize;
1330  if (auto Fragment = Expr->getFragmentInfo())
1331  BitsToDescribe = Fragment->SizeInBits;
1332  for (auto RegAndSize : RFV.getRegsAndSizes()) {
1333  unsigned RegisterSize = RegAndSize.second;
1334  // Bail out if all bits are described already.
1335  if (Offset >= BitsToDescribe)
1336  break;
1337  unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1338  ? BitsToDescribe - Offset
1339  : RegisterSize;
1340  auto FragmentExpr = DIExpression::createFragmentExpression(
1341  Expr, Offset, FragmentSize);
1342  if (!FragmentExpr)
1343  continue;
1344  SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
1345  false, dl, SDNodeOrder);
1346  DAG.AddDbgValue(SDV, nullptr, false);
1347  Offset += RegisterSize;
1348  }
1349  } else {
1350  SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
1351  DAG.AddDbgValue(SDV, nullptr, false);
1352  }
1353  return true;
1354  }
1355  }
1356 
1357  return false;
1358 }
1359 
1361  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1362  for (auto &Pair : DanglingDebugInfoMap)
1363  for (auto &DDI : Pair.second)
1364  salvageUnresolvedDbgValue(DDI);
1365  clearDanglingDebugInfo();
1366 }
1367 
1368 /// getCopyFromRegs - If there was virtual register allocated for the value V
1369 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1371  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1372  SDValue Result;
1373 
1374  if (It != FuncInfo.ValueMap.end()) {
1375  unsigned InReg = It->second;
1376 
1377  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1378  DAG.getDataLayout(), InReg, Ty,
1379  None); // This is not an ABI copy.
1380  SDValue Chain = DAG.getEntryNode();
1381  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1382  V);
1383  resolveDanglingDebugInfo(V, Result);
1384  }
1385 
1386  return Result;
1387 }
1388 
1389 /// getValue - Return an SDValue for the given Value.
1391  // If we already have an SDValue for this value, use it. It's important
1392  // to do this first, so that we don't create a CopyFromReg if we already
1393  // have a regular SDValue.
1394  SDValue &N = NodeMap[V];
1395  if (N.getNode()) return N;
1396 
1397  // If there's a virtual register allocated and initialized for this
1398  // value, use it.
1399  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1400  return copyFromReg;
1401 
1402  // Otherwise create a new SDValue and remember it.
1403  SDValue Val = getValueImpl(V);
1404  NodeMap[V] = Val;
1405  resolveDanglingDebugInfo(V, Val);
1406  return Val;
1407 }
1408 
1409 // Return true if SDValue exists for the given Value
1411  return (NodeMap.find(V) != NodeMap.end()) ||
1412  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1413 }
1414 
1415 /// getNonRegisterValue - Return an SDValue for the given Value, but
1416 /// don't look in FuncInfo.ValueMap for a virtual register.
1418  // If we already have an SDValue for this value, use it.
1419  SDValue &N = NodeMap[V];
1420  if (N.getNode()) {
1421  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1422  // Remove the debug location from the node as the node is about to be used
1423  // in a location which may differ from the original debug location. This
1424  // is relevant to Constant and ConstantFP nodes because they can appear
1425  // as constant expressions inside PHI nodes.
1426  N->setDebugLoc(DebugLoc());
1427  }
1428  return N;
1429  }
1430 
1431  // Otherwise create a new SDValue and remember it.
1432  SDValue Val = getValueImpl(V);
1433  NodeMap[V] = Val;
1434  resolveDanglingDebugInfo(V, Val);
1435  return Val;
1436 }
1437 
1438 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1439 /// Create an SDValue for the given value.
1441  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1442 
1443  if (const Constant *C = dyn_cast<Constant>(V)) {
1444  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1445 
1446  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1447  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1448 
1449  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1450  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1451 
1452  if (isa<ConstantPointerNull>(C)) {
1453  unsigned AS = V->getType()->getPointerAddressSpace();
1454  return DAG.getConstant(0, getCurSDLoc(),
1455  TLI.getPointerTy(DAG.getDataLayout(), AS));
1456  }
1457 
1458  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1459  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1460 
1461  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1462  return DAG.getUNDEF(VT);
1463 
1464  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1465  visit(CE->getOpcode(), *CE);
1466  SDValue N1 = NodeMap[V];
1467  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1468  return N1;
1469  }
1470 
1471  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1473  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1474  OI != OE; ++OI) {
1475  SDNode *Val = getValue(*OI).getNode();
1476  // If the operand is an empty aggregate, there are no values.
1477  if (!Val) continue;
1478  // Add each leaf value from the operand to the Constants list
1479  // to form a flattened list of all the values.
1480  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1481  Constants.push_back(SDValue(Val, i));
1482  }
1483 
1484  return DAG.getMergeValues(Constants, getCurSDLoc());
1485  }
1486 
1487  if (const ConstantDataSequential *CDS =
1488  dyn_cast<ConstantDataSequential>(C)) {
1490  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1491  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1492  // Add each leaf value from the operand to the Constants list
1493  // to form a flattened list of all the values.
1494  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1495  Ops.push_back(SDValue(Val, i));
1496  }
1497 
1498  if (isa<ArrayType>(CDS->getType()))
1499  return DAG.getMergeValues(Ops, getCurSDLoc());
1500  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1501  }
1502 
1503  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1504  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1505  "Unknown struct or array constant!");
1506 
1508  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1509  unsigned NumElts = ValueVTs.size();
1510  if (NumElts == 0)
1511  return SDValue(); // empty struct
1513  for (unsigned i = 0; i != NumElts; ++i) {
1514  EVT EltVT = ValueVTs[i];
1515  if (isa<UndefValue>(C))
1516  Constants[i] = DAG.getUNDEF(EltVT);
1517  else if (EltVT.isFloatingPoint())
1518  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1519  else
1520  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1521  }
1522 
1523  return DAG.getMergeValues(Constants, getCurSDLoc());
1524  }
1525 
1526  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1527  return DAG.getBlockAddress(BA, VT);
1528 
1529  VectorType *VecTy = cast<VectorType>(V->getType());
1530  unsigned NumElements = VecTy->getNumElements();
1531 
1532  // Now that we know the number and type of the elements, get that number of
1533  // elements into the Ops array based on what kind of constant it is.
1535  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1536  for (unsigned i = 0; i != NumElements; ++i)
1537  Ops.push_back(getValue(CV->getOperand(i)));
1538  } else {
1539  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1540  EVT EltVT =
1541  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1542 
1543  SDValue Op;
1544  if (EltVT.isFloatingPoint())
1545  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1546  else
1547  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1548  Ops.assign(NumElements, Op);
1549  }
1550 
1551  // Create a BUILD_VECTOR node.
1552  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1553  }
1554 
1555  // If this is a static alloca, generate it as the frameindex instead of
1556  // computation.
1557  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1559  FuncInfo.StaticAllocaMap.find(AI);
1560  if (SI != FuncInfo.StaticAllocaMap.end())
1561  return DAG.getFrameIndex(SI->second,
1562  TLI.getFrameIndexTy(DAG.getDataLayout()));
1563  }
1564 
1565  // If this is an instruction which fast-isel has deferred, select it now.
1566  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1567  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1568 
1569  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1570  Inst->getType(), getABIRegCopyCC(V));
1571  SDValue Chain = DAG.getEntryNode();
1572  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1573  }
1574 
1575  llvm_unreachable("Can't get register for value!");
1576 }
1577 
1578 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1579  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1580  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1581  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1582  bool IsSEH = isAsynchronousEHPersonality(Pers);
1583  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1584  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1585  if (!IsSEH)
1586  CatchPadMBB->setIsEHScopeEntry();
1587  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1588  if (IsMSVCCXX || IsCoreCLR)
1589  CatchPadMBB->setIsEHFuncletEntry();
1590  // Wasm does not need catchpads anymore
1591  if (!IsWasmCXX)
1592  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1593  getControlRoot()));
1594 }
1595 
1596 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1597  // Update machine-CFG edge.
1598  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1599  FuncInfo.MBB->addSuccessor(TargetMBB);
1600 
1601  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1602  bool IsSEH = isAsynchronousEHPersonality(Pers);
1603  if (IsSEH) {
1604  // If this is not a fall-through branch or optimizations are switched off,
1605  // emit the branch.
1606  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1607  TM.getOptLevel() == CodeGenOpt::None)
1608  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1609  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1610  return;
1611  }
1612 
1613  // Figure out the funclet membership for the catchret's successor.
1614  // This will be used by the FuncletLayout pass to determine how to order the
1615  // BB's.
1616  // A 'catchret' returns to the outer scope's color.
1617  Value *ParentPad = I.getCatchSwitchParentPad();
1618  const BasicBlock *SuccessorColor;
1619  if (isa<ConstantTokenNone>(ParentPad))
1620  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1621  else
1622  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1623  assert(SuccessorColor && "No parent funclet for catchret!");
1624  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1625  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1626 
1627  // Create the terminator node.
1628  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1629  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1630  DAG.getBasicBlock(SuccessorColorMBB));
1631  DAG.setRoot(Ret);
1632 }
1633 
1634 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1635  // Don't emit any special code for the cleanuppad instruction. It just marks
1636  // the start of an EH scope/funclet.
1637  FuncInfo.MBB->setIsEHScopeEntry();
1638  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1639  if (Pers != EHPersonality::Wasm_CXX) {
1640  FuncInfo.MBB->setIsEHFuncletEntry();
1641  FuncInfo.MBB->setIsCleanupFuncletEntry();
1642  }
1643 }
1644 
1645 // For wasm, there's alwyas a single catch pad attached to a catchswitch, and
1646 // the control flow always stops at the single catch pad, as it does for a
1647 // cleanup pad. In case the exception caught is not of the types the catch pad
1648 // catches, it will be rethrown by a rethrow.
1650  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1651  BranchProbability Prob,
1652  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1653  &UnwindDests) {
1654  while (EHPadBB) {
1655  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1656  if (isa<CleanupPadInst>(Pad)) {
1657  // Stop on cleanup pads.
1658  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1659  UnwindDests.back().first->setIsEHScopeEntry();
1660  break;
1661  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1662  // Add the catchpad handlers to the possible destinations. We don't
1663  // continue to the unwind destination of the catchswitch for wasm.
1664  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1665  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1666  UnwindDests.back().first->setIsEHScopeEntry();
1667  }
1668  break;
1669  } else {
1670  continue;
1671  }
1672  }
1673 }
1674 
1675 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1676 /// many places it could ultimately go. In the IR, we have a single unwind
1677 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1678 /// This function skips over imaginary basic blocks that hold catchswitch
1679 /// instructions, and finds all the "real" machine
1680 /// basic block destinations. As those destinations may not be successors of
1681 /// EHPadBB, here we also calculate the edge probability to those destinations.
1682 /// The passed-in Prob is the edge probability to EHPadBB.
1684  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1685  BranchProbability Prob,
1686  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1687  &UnwindDests) {
1688  EHPersonality Personality =
1690  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1691  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1692  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1693  bool IsSEH = isAsynchronousEHPersonality(Personality);
1694 
1695  if (IsWasmCXX) {
1696  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1697  assert(UnwindDests.size() <= 1 &&
1698  "There should be at most one unwind destination for wasm");
1699  return;
1700  }
1701 
1702  while (EHPadBB) {
1703  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1704  BasicBlock *NewEHPadBB = nullptr;
1705  if (isa<LandingPadInst>(Pad)) {
1706  // Stop on landingpads. They are not funclets.
1707  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1708  break;
1709  } else if (isa<CleanupPadInst>(Pad)) {
1710  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1711  // personalities.
1712  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1713  UnwindDests.back().first->setIsEHScopeEntry();
1714  UnwindDests.back().first->setIsEHFuncletEntry();
1715  break;
1716  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1717  // Add the catchpad handlers to the possible destinations.
1718  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1719  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1720  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1721  if (IsMSVCCXX || IsCoreCLR)
1722  UnwindDests.back().first->setIsEHFuncletEntry();
1723  if (!IsSEH)
1724  UnwindDests.back().first->setIsEHScopeEntry();
1725  }
1726  NewEHPadBB = CatchSwitch->getUnwindDest();
1727  } else {
1728  continue;
1729  }
1730 
1731  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1732  if (BPI && NewEHPadBB)
1733  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1734  EHPadBB = NewEHPadBB;
1735  }
1736 }
1737 
1738 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1739  // Update successor info.
1741  auto UnwindDest = I.getUnwindDest();
1742  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1743  BranchProbability UnwindDestProb =
1744  (BPI && UnwindDest)
1745  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1747  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1748  for (auto &UnwindDest : UnwindDests) {
1749  UnwindDest.first->setIsEHPad();
1750  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1751  }
1752  FuncInfo.MBB->normalizeSuccProbs();
1753 
1754  // Create the terminator node.
1755  SDValue Ret =
1756  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1757  DAG.setRoot(Ret);
1758 }
1759 
1760 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1761  report_fatal_error("visitCatchSwitch not yet implemented!");
1762 }
1763 
1764 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1765  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1766  auto &DL = DAG.getDataLayout();
1767  SDValue Chain = getControlRoot();
1769  SmallVector<SDValue, 8> OutVals;
1770 
1771  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1772  // lower
1773  //
1774  // %val = call <ty> @llvm.experimental.deoptimize()
1775  // ret <ty> %val
1776  //
1777  // differently.
1779  LowerDeoptimizingReturn();
1780  return;
1781  }
1782 
1783  if (!FuncInfo.CanLowerReturn) {
1784  unsigned DemoteReg = FuncInfo.DemoteRegister;
1785  const Function *F = I.getParent()->getParent();
1786 
1787  // Emit a store of the return value through the virtual register.
1788  // Leave Outs empty so that LowerReturn won't try to load return
1789  // registers the usual way.
1790  SmallVector<EVT, 1> PtrValueVTs;
1791  ComputeValueVTs(TLI, DL,
1794  PtrValueVTs);
1795 
1796  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1797  DemoteReg, PtrValueVTs[0]);
1798  SDValue RetOp = getValue(I.getOperand(0));
1799 
1800  SmallVector<EVT, 4> ValueVTs, MemVTs;
1802  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1803  &Offsets);
1804  unsigned NumValues = ValueVTs.size();
1805 
1806  SmallVector<SDValue, 4> Chains(NumValues);
1807  for (unsigned i = 0; i != NumValues; ++i) {
1808  // An aggregate return value cannot wrap around the address space, so
1809  // offsets to its parts don't wrap either.
1810  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1811 
1812  SDValue Val = RetOp.getValue(i);
1813  if (MemVTs[i] != ValueVTs[i])
1814  Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1815  Chains[i] = DAG.getStore(Chain, getCurSDLoc(), Val,
1816  // FIXME: better loc info would be nice.
1818  }
1819 
1820  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1821  MVT::Other, Chains);
1822  } else if (I.getNumOperands() != 0) {
1824  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1825  unsigned NumValues = ValueVTs.size();
1826  if (NumValues) {
1827  SDValue RetOp = getValue(I.getOperand(0));
1828 
1829  const Function *F = I.getParent()->getParent();
1830 
1831  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1832  I.getOperand(0)->getType(), F->getCallingConv(),
1833  /*IsVarArg*/ false);
1834 
1835  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1836  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1837  Attribute::SExt))
1838  ExtendKind = ISD::SIGN_EXTEND;
1839  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1840  Attribute::ZExt))
1841  ExtendKind = ISD::ZERO_EXTEND;
1842 
1843  LLVMContext &Context = F->getContext();
1844  bool RetInReg = F->getAttributes().hasAttribute(
1845  AttributeList::ReturnIndex, Attribute::InReg);
1846 
1847  for (unsigned j = 0; j != NumValues; ++j) {
1848  EVT VT = ValueVTs[j];
1849 
1850  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1851  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1852 
1853  CallingConv::ID CC = F->getCallingConv();
1854 
1855  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1856  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1857  SmallVector<SDValue, 4> Parts(NumParts);
1858  getCopyToParts(DAG, getCurSDLoc(),
1859  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1860  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1861 
1862  // 'inreg' on function refers to return value
1863  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1864  if (RetInReg)
1865  Flags.setInReg();
1866 
1867  if (I.getOperand(0)->getType()->isPointerTy()) {
1868  Flags.setPointer();
1869  Flags.setPointerAddrSpace(
1870  cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
1871  }
1872 
1873  if (NeedsRegBlock) {
1874  Flags.setInConsecutiveRegs();
1875  if (j == NumValues - 1)
1876  Flags.setInConsecutiveRegsLast();
1877  }
1878 
1879  // Propagate extension type if any
1880  if (ExtendKind == ISD::SIGN_EXTEND)
1881  Flags.setSExt();
1882  else if (ExtendKind == ISD::ZERO_EXTEND)
1883  Flags.setZExt();
1884 
1885  for (unsigned i = 0; i < NumParts; ++i) {
1886  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1887  VT, /*isfixed=*/true, 0, 0));
1888  OutVals.push_back(Parts[i]);
1889  }
1890  }
1891  }
1892  }
1893 
1894  // Push in swifterror virtual register as the last element of Outs. This makes
1895  // sure swifterror virtual register will be returned in the swifterror
1896  // physical register.
1897  const Function *F = I.getParent()->getParent();
1898  if (TLI.supportSwiftError() &&
1899  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1900  assert(SwiftError.getFunctionArg() && "Need a swift error argument");
1901  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1902  Flags.setSwiftError();
1903  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1904  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1905  true /*isfixed*/, 1 /*origidx*/,
1906  0 /*partOffs*/));
1907  // Create SDNode for the swifterror virtual register.
1908  OutVals.push_back(
1909  DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
1910  &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
1911  EVT(TLI.getPointerTy(DL))));
1912  }
1913 
1914  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1915  CallingConv::ID CallConv =
1917  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1918  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1919 
1920  // Verify that the target's LowerReturn behaved as expected.
1921  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1922  "LowerReturn didn't return a valid chain!");
1923 
1924  // Update the DAG with the new chain value resulting from return lowering.
1925  DAG.setRoot(Chain);
1926 }
1927 
1928 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1929 /// created for it, emit nodes to copy the value into the virtual
1930 /// registers.
1932  // Skip empty types
1933  if (V->getType()->isEmptyTy())
1934  return;
1935 
1936  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1937  if (VMI != FuncInfo.ValueMap.end()) {
1938  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1939  CopyValueToVirtualRegister(V, VMI->second);
1940  }
1941 }
1942 
1943 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1944 /// the current basic block, add it to ValueMap now so that we'll get a
1945 /// CopyTo/FromReg.
1947  // No need to export constants.
1948  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1949 
1950  // Already exported?
1951  if (FuncInfo.isExportedInst(V)) return;
1952 
1953  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1954  CopyValueToVirtualRegister(V, Reg);
1955 }
1956 
1958  const BasicBlock *FromBB) {
1959  // The operands of the setcc have to be in this block. We don't know
1960  // how to export them from some other block.
1961  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1962  // Can export from current BB.
1963  if (VI->getParent() == FromBB)
1964  return true;
1965 
1966  // Is already exported, noop.
1967  return FuncInfo.isExportedInst(V);
1968  }
1969 
1970  // If this is an argument, we can export it if the BB is the entry block or
1971  // if it is already exported.
1972  if (isa<Argument>(V)) {
1973  if (FromBB == &FromBB->getParent()->getEntryBlock())
1974  return true;
1975 
1976  // Otherwise, can only export this if it is already exported.
1977  return FuncInfo.isExportedInst(V);
1978  }
1979 
1980  // Otherwise, constants can always be exported.
1981  return true;
1982 }
1983 
1984 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1986 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1987  const MachineBasicBlock *Dst) const {
1988  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1989  const BasicBlock *SrcBB = Src->getBasicBlock();
1990  const BasicBlock *DstBB = Dst->getBasicBlock();
1991  if (!BPI) {
1992  // If BPI is not available, set the default probability as 1 / N, where N is
1993  // the number of successors.
1994  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1995  return BranchProbability(1, SuccSize);
1996  }
1997  return BPI->getEdgeProbability(SrcBB, DstBB);
1998 }
1999 
2000 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2001  MachineBasicBlock *Dst,
2002  BranchProbability Prob) {
2003  if (!FuncInfo.BPI)
2004  Src->addSuccessorWithoutProb(Dst);
2005  else {
2006  if (Prob.isUnknown())
2007  Prob = getEdgeProbability(Src, Dst);
2008  Src->addSuccessor(Dst, Prob);
2009  }
2010 }
2011 
2012 static bool InBlock(const Value *V, const BasicBlock *BB) {
2013  if (const Instruction *I = dyn_cast<Instruction>(V))
2014  return I->getParent() == BB;
2015  return true;
2016 }
2017 
2018 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2019 /// This function emits a branch and is used at the leaves of an OR or an
2020 /// AND operator tree.
2021 void
2023  MachineBasicBlock *TBB,
2024  MachineBasicBlock *FBB,
2025  MachineBasicBlock *CurBB,
2026  MachineBasicBlock *SwitchBB,
2027  BranchProbability TProb,
2028  BranchProbability FProb,
2029  bool InvertCond) {
2030  const BasicBlock *BB = CurBB->getBasicBlock();
2031 
2032  // If the leaf of the tree is a comparison, merge the condition into
2033  // the caseblock.
2034  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2035  // The operands of the cmp have to be in this block. We don't know
2036  // how to export them from some other block. If this is the first block
2037  // of the sequence, no exporting is needed.
2038  if (CurBB == SwitchBB ||
2039  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2040  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2041  ISD::CondCode Condition;
2042  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2043  ICmpInst::Predicate Pred =
2044  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2045  Condition = getICmpCondCode(Pred);
2046  } else {
2047  const FCmpInst *FC = cast<FCmpInst>(Cond);
2048  FCmpInst::Predicate Pred =
2049  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2050  Condition = getFCmpCondCode(Pred);
2051  if (TM.Options.NoNaNsFPMath)
2052  Condition = getFCmpCodeWithoutNaN(Condition);
2053  }
2054 
2055  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2056  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2057  SL->SwitchCases.push_back(CB);
2058  return;
2059  }
2060  }
2061 
2062  // Create a CaseBlock record representing this branch.
2063  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2064  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2065  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2066  SL->SwitchCases.push_back(CB);
2067 }
2068 
2070  MachineBasicBlock *TBB,
2071  MachineBasicBlock *FBB,
2072  MachineBasicBlock *CurBB,
2073  MachineBasicBlock *SwitchBB,
2075  BranchProbability TProb,
2076  BranchProbability FProb,
2077  bool InvertCond) {
2078  // Skip over not part of the tree and remember to invert op and operands at
2079  // next level.
2080  Value *NotCond;
2081  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2082  InBlock(NotCond, CurBB->getBasicBlock())) {
2083  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2084  !InvertCond);
2085  return;
2086  }
2087 
2088  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2089  // Compute the effective opcode for Cond, taking into account whether it needs
2090  // to be inverted, e.g.
2091  // and (not (or A, B)), C
2092  // gets lowered as
2093  // and (and (not A, not B), C)
2094  unsigned BOpc = 0;
2095  if (BOp) {
2096  BOpc = BOp->getOpcode();
2097  if (InvertCond) {
2098  if (BOpc == Instruction::And)
2099  BOpc = Instruction::Or;
2100  else if (BOpc == Instruction::Or)
2101  BOpc = Instruction::And;
2102  }
2103  }
2104 
2105  // If this node is not part of the or/and tree, emit it as a branch.
2106  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
2107  BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
2108  BOp->getParent() != CurBB->getBasicBlock() ||
2109  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
2110  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
2111  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2112  TProb, FProb, InvertCond);
2113  return;
2114  }
2115 
2116  // Create TmpBB after CurBB.
2117  MachineFunction::iterator BBI(CurBB);
2118  MachineFunction &MF = DAG.getMachineFunction();
2120  CurBB->getParent()->insert(++BBI, TmpBB);
2121 
2122  if (Opc == Instruction::Or) {
2123  // Codegen X | Y as:
2124  // BB1:
2125  // jmp_if_X TBB
2126  // jmp TmpBB
2127  // TmpBB:
2128  // jmp_if_Y TBB
2129  // jmp FBB
2130  //
2131 
2132  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2133  // The requirement is that
2134  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2135  // = TrueProb for original BB.
2136  // Assuming the original probabilities are A and B, one choice is to set
2137  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2138  // A/(1+B) and 2B/(1+B). This choice assumes that
2139  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2140  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2141  // TmpBB, but the math is more complicated.
2142 
2143  auto NewTrueProb = TProb / 2;
2144  auto NewFalseProb = TProb / 2 + FProb;
2145  // Emit the LHS condition.
2146  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
2147  NewTrueProb, NewFalseProb, InvertCond);
2148 
2149  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2150  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2151  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2152  // Emit the RHS condition into TmpBB.
2153  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2154  Probs[0], Probs[1], InvertCond);
2155  } else {
2156  assert(Opc == Instruction::And && "Unknown merge op!");
2157  // Codegen X & Y as:
2158  // BB1:
2159  // jmp_if_X TmpBB
2160  // jmp FBB
2161  // TmpBB:
2162  // jmp_if_Y TBB
2163  // jmp FBB
2164  //
2165  // This requires creation of TmpBB after CurBB.
2166 
2167  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2168  // The requirement is that
2169  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2170  // = FalseProb for original BB.
2171  // Assuming the original probabilities are A and B, one choice is to set
2172  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2173  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2174  // TrueProb for BB1 * FalseProb for TmpBB.
2175 
2176  auto NewTrueProb = TProb + FProb / 2;
2177  auto NewFalseProb = FProb / 2;
2178  // Emit the LHS condition.
2179  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
2180  NewTrueProb, NewFalseProb, InvertCond);
2181 
2182  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2183  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2184  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2185  // Emit the RHS condition into TmpBB.
2186  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2187  Probs[0], Probs[1], InvertCond);
2188  }
2189 }
2190 
2191 /// If the set of cases should be emitted as a series of branches, return true.
2192 /// If we should emit this as a bunch of and/or'd together conditions, return
2193 /// false.
2194 bool
2195 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2196  if (Cases.size() != 2) return true;
2197 
2198  // If this is two comparisons of the same values or'd or and'd together, they
2199  // will get folded into a single comparison, so don't emit two blocks.
2200  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2201  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2202  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2203  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2204  return false;
2205  }
2206 
2207  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2208  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2209  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2210  Cases[0].CC == Cases[1].CC &&
2211  isa<Constant>(Cases[0].CmpRHS) &&
2212  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2213  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2214  return false;
2215  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2216  return false;
2217  }
2218 
2219  return true;
2220 }
2221 
2222 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2223  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2224 
2225  // Update machine-CFG edges.
2226  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2227 
2228  if (I.isUnconditional()) {
2229  // Update machine-CFG edges.
2230  BrMBB->addSuccessor(Succ0MBB);
2231 
2232  // If this is not a fall-through branch or optimizations are switched off,
2233  // emit the branch.
2234  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2235  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2236  MVT::Other, getControlRoot(),
2237  DAG.getBasicBlock(Succ0MBB)));
2238 
2239  return;
2240  }
2241 
2242  // If this condition is one of the special cases we handle, do special stuff
2243  // now.
2244  const Value *CondVal = I.getCondition();
2245  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2246 
2247  // If this is a series of conditions that are or'd or and'd together, emit
2248  // this as a sequence of branches instead of setcc's with and/or operations.
2249  // As long as jumps are not expensive, this should improve performance.
2250  // For example, instead of something like:
2251  // cmp A, B
2252  // C = seteq
2253  // cmp D, E
2254  // F = setle
2255  // or C, F
2256  // jnz foo
2257  // Emit:
2258  // cmp A, B
2259  // je foo
2260  // cmp D, E
2261  // jle foo
2262  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2263  Instruction::BinaryOps Opcode = BOp->getOpcode();
2264  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2265  !I.getMetadata(LLVMContext::MD_unpredictable) &&
2266  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2267  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2268  Opcode,
2269  getEdgeProbability(BrMBB, Succ0MBB),
2270  getEdgeProbability(BrMBB, Succ1MBB),
2271  /*InvertCond=*/false);
2272  // If the compares in later blocks need to use values not currently
2273  // exported from this block, export them now. This block should always
2274  // be the first entry.
2275  assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2276 
2277  // Allow some cases to be rejected.
2278  if (ShouldEmitAsBranches(SL->SwitchCases)) {
2279  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2280  ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2281  ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2282  }
2283 
2284  // Emit the branch for this block.
2285  visitSwitchCase(SL->SwitchCases[0], BrMBB);
2286  SL->SwitchCases.erase(SL->SwitchCases.begin());
2287  return;
2288  }
2289 
2290  // Okay, we decided not to do this, remove any inserted MBB's and clear
2291  // SwitchCases.
2292  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2293  FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2294 
2295  SL->SwitchCases.clear();
2296  }
2297  }
2298 
2299  // Create a CaseBlock record representing this branch.
2300  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2301  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2302 
2303  // Use visitSwitchCase to actually insert the fast branch sequence for this
2304  // cond branch.
2305  visitSwitchCase(CB, BrMBB);
2306 }
2307 
2308 /// visitSwitchCase - Emits the necessary code to represent a single node in
2309 /// the binary search tree resulting from lowering a switch instruction.
2311  MachineBasicBlock *SwitchBB) {
2312  SDValue Cond;
2313  SDValue CondLHS = getValue(CB.CmpLHS);
2314  SDLoc dl = CB.DL;
2315 
2316  if (CB.CC == ISD::SETTRUE) {
2317  // Branch or fall through to TrueBB.
2318  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2319  SwitchBB->normalizeSuccProbs();
2320  if (CB.TrueBB != NextBlock(SwitchBB)) {
2321  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2322  DAG.getBasicBlock(CB.TrueBB)));
2323  }
2324  return;
2325  }
2326 
2327  auto &TLI = DAG.getTargetLoweringInfo();
2328  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2329 
2330  // Build the setcc now.
2331  if (!CB.CmpMHS) {
2332  // Fold "(X == true)" to X and "(X == false)" to !X to
2333  // handle common cases produced by branch lowering.
2334  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2335  CB.CC == ISD::SETEQ)
2336  Cond = CondLHS;
2337  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2338  CB.CC == ISD::SETEQ) {
2339  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2340  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2341  } else {
2342  SDValue CondRHS = getValue(CB.CmpRHS);
2343 
2344  // If a pointer's DAG type is larger than its memory type then the DAG
2345  // values are zero-extended. This breaks signed comparisons so truncate
2346  // back to the underlying type before doing the compare.
2347  if (CondLHS.getValueType() != MemVT) {
2348  CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2349  CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2350  }
2351  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2352  }
2353  } else {
2354  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2355 
2356  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2357  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2358 
2359  SDValue CmpOp = getValue(CB.CmpMHS);
2360  EVT VT = CmpOp.getValueType();
2361 
2362  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2363  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2364  ISD::SETLE);
2365  } else {
2366  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2367  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2368  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2369  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2370  }
2371  }
2372 
2373  // Update successor info
2374  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2375  // TrueBB and FalseBB are always different unless the incoming IR is
2376  // degenerate. This only happens when running llc on weird IR.
2377  if (CB.TrueBB != CB.FalseBB)
2378  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2379  SwitchBB->normalizeSuccProbs();
2380 
2381  // If the lhs block is the next block, invert the condition so that we can
2382  // fall through to the lhs instead of the rhs block.
2383  if (CB.TrueBB == NextBlock(SwitchBB)) {
2384  std::swap(CB.TrueBB, CB.FalseBB);
2385  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2386  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2387  }
2388 
2389  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2390  MVT::Other, getControlRoot(), Cond,
2391  DAG.getBasicBlock(CB.TrueBB));
2392 
2393  // Insert the false branch. Do this even if it's a fall through branch,
2394  // this makes it easier to do DAG optimizations which require inverting
2395  // the branch condition.
2396  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2397  DAG.getBasicBlock(CB.FalseBB));
2398 
2399  DAG.setRoot(BrCond);
2400 }
2401 
2402 /// visitJumpTable - Emit JumpTable node in the current MBB
2404  // Emit the code for the jump table
2405  assert(JT.Reg != -1U && "Should lower JT Header first!");
2407  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2408  JT.Reg, PTy);
2409  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2410  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2411  MVT::Other, Index.getValue(1),
2412  Table, Index);
2413  DAG.setRoot(BrJumpTable);
2414 }
2415 
2416 /// visitJumpTableHeader - This function emits necessary code to produce index
2417 /// in the JumpTable from switch case.
2419  JumpTableHeader &JTH,
2420  MachineBasicBlock *SwitchBB) {
2421  SDLoc dl = getCurSDLoc();
2422 
2423  // Subtract the lowest switch case value from the value being switched on.
2424  SDValue SwitchOp = getValue(JTH.SValue);
2425  EVT VT = SwitchOp.getValueType();
2426  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2427  DAG.getConstant(JTH.First, dl, VT));
2428 
2429  // The SDNode we just created, which holds the value being switched on minus
2430  // the smallest case value, needs to be copied to a virtual register so it
2431  // can be used as an index into the jump table in a subsequent basic block.
2432  // This value may be smaller or larger than the target's pointer type, and
2433  // therefore require extension or truncating.
2434  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2435  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2436 
2437  unsigned JumpTableReg =
2438  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2439  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2440  JumpTableReg, SwitchOp);
2441  JT.Reg = JumpTableReg;
2442 
2443  if (!JTH.OmitRangeCheck) {
2444  // Emit the range check for the jump table, and branch to the default block
2445  // for the switch statement if the value being switched on exceeds the
2446  // largest case in the switch.
2447  SDValue CMP = DAG.getSetCC(
2448  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2449  Sub.getValueType()),
2450  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2451 
2452  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2453  MVT::Other, CopyTo, CMP,
2454  DAG.getBasicBlock(JT.Default));
2455 
2456  // Avoid emitting unnecessary branches to the next block.
2457  if (JT.MBB != NextBlock(SwitchBB))
2458  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2459  DAG.getBasicBlock(JT.MBB));
2460 
2461  DAG.setRoot(BrCond);
2462  } else {
2463  // Avoid emitting unnecessary branches to the next block.
2464  if (JT.MBB != NextBlock(SwitchBB))
2465  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2466  DAG.getBasicBlock(JT.MBB)));
2467  else
2468  DAG.setRoot(CopyTo);
2469  }
2470 }
2471 
2472 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2473 /// variable if there exists one.
2475  SDValue &Chain) {
2476  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2477  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2478  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2479  MachineFunction &MF = DAG.getMachineFunction();
2480  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2481  MachineSDNode *Node =
2482  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2483  if (Global) {
2484  MachinePointerInfo MPInfo(Global);
2488  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2489  DAG.setNodeMemRefs(Node, {MemRef});
2490  }
2491  if (PtrTy != PtrMemTy)
2492  return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2493  return SDValue(Node, 0);
2494 }
2495 
2496 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2497 /// tail spliced into a stack protector check success bb.
2498 ///
2499 /// For a high level explanation of how this fits into the stack protector
2500 /// generation see the comment on the declaration of class
2501 /// StackProtectorDescriptor.
2502 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2503  MachineBasicBlock *ParentBB) {
2504 
2505  // First create the loads to the guard/stack slot for the comparison.
2506  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2507  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2508  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2509 
2510  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2511  int FI = MFI.getStackProtectorIndex();
2512 
2513  SDValue Guard;
2514  SDLoc dl = getCurSDLoc();
2515  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2516  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2517  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2518 
2519  // Generate code to load the content of the guard slot.
2520  SDValue GuardVal = DAG.getLoad(
2521  PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2524 
2525  if (TLI.useStackGuardXorFP())
2526  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2527 
2528  // Retrieve guard check function, nullptr if instrumentation is inlined.
2529  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2530  // The target provides a guard check function to validate the guard value.
2531  // Generate a call to that function with the content of the guard slot as
2532  // argument.
2533  FunctionType *FnTy = GuardCheckFn->getFunctionType();
2534  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2535 
2538  Entry.Node = GuardVal;
2539  Entry.Ty = FnTy->getParamType(0);
2540  if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
2541  Entry.IsInReg = true;
2542  Args.push_back(Entry);
2543 
2545  CLI.setDebugLoc(getCurSDLoc())
2546  .setChain(DAG.getEntryNode())
2547  .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2548  getValue(GuardCheckFn), std::move(Args));
2549 
2550  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2551  DAG.setRoot(Result.second);
2552  return;
2553  }
2554 
2555  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2556  // Otherwise, emit a volatile load to retrieve the stack guard value.
2557  SDValue Chain = DAG.getEntryNode();
2558  if (TLI.useLoadStackGuardNode()) {
2559  Guard = getLoadStackGuard(DAG, dl, Chain);
2560  } else {
2561  const Value *IRGuard = TLI.getSDagStackGuard(M);
2562  SDValue GuardPtr = getValue(IRGuard);
2563 
2564  Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2565  MachinePointerInfo(IRGuard, 0), Align,
2567  }
2568 
2569  // Perform the comparison via a subtract/getsetcc.
2570  EVT VT = Guard.getValueType();
2571  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2572 
2573  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2574  *DAG.getContext(),
2575  Sub.getValueType()),
2576  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2577 
2578  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2579  // branch to failure MBB.
2580  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2581  MVT::Other, GuardVal.getOperand(0),
2582  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2583  // Otherwise branch to success MBB.
2584  SDValue Br = DAG.getNode(ISD::BR, dl,
2585  MVT::Other, BrCond,
2586  DAG.getBasicBlock(SPD.getSuccessMBB()));
2587 
2588  DAG.setRoot(Br);
2589 }
2590 
2591 /// Codegen the failure basic block for a stack protector check.
2592 ///
2593 /// A failure stack protector machine basic block consists simply of a call to
2594 /// __stack_chk_fail().
2595 ///
2596 /// For a high level explanation of how this fits into the stack protector
2597 /// generation see the comment on the declaration of class
2598 /// StackProtectorDescriptor.
2599 void
2600 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2601  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2602  SDValue Chain =
2603  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2604  None, false, getCurSDLoc(), false, false).second;
2605  // On PS4, the "return address" must still be within the calling function,
2606  // even if it's at the very end, so emit an explicit TRAP here.
2607  // Passing 'true' for doesNotReturn above won't generate the trap for us.
2608  if (TM.getTargetTriple().isPS4CPU())
2609  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2610 
2611  DAG.setRoot(Chain);
2612 }
2613 
2614 /// visitBitTestHeader - This function emits necessary code to produce value
2615 /// suitable for "bit tests"
2617  MachineBasicBlock *SwitchBB) {
2618  SDLoc dl = getCurSDLoc();
2619 
2620  // Subtract the minimum value
2621  SDValue SwitchOp = getValue(B.SValue);
2622  EVT VT = SwitchOp.getValueType();
2623  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2624  DAG.getConstant(B.First, dl, VT));
2625 
2626  // Check range
2627  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2628  SDValue RangeCmp = DAG.getSetCC(
2629  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2630  Sub.getValueType()),
2631  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2632 
2633  // Determine the type of the test operands.
2634  bool UsePtrType = false;
2635  if (!TLI.isTypeLegal(VT))
2636  UsePtrType = true;
2637  else {
2638  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2639  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2640  // Switch table case range are encoded into series of masks.
2641  // Just use pointer type, it's guaranteed to fit.
2642  UsePtrType = true;
2643  break;
2644  }
2645  }
2646  if (UsePtrType) {
2647  VT = TLI.getPointerTy(DAG.getDataLayout());
2648  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2649  }
2650 
2651  B.RegVT = VT.getSimpleVT();
2652  B.Reg = FuncInfo.CreateReg(B.RegVT);
2653  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2654 
2655  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2656 
2657  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2658  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2659  SwitchBB->normalizeSuccProbs();
2660 
2661  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2662  MVT::Other, CopyTo, RangeCmp,
2663  DAG.getBasicBlock(B.Default));
2664 
2665  // Avoid emitting unnecessary branches to the next block.
2666  if (MBB != NextBlock(SwitchBB))
2667  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2668  DAG.getBasicBlock(MBB));
2669 
2670  DAG.setRoot(BrRange);
2671 }
2672 
2673 /// visitBitTestCase - this function produces one "bit test"
2675  MachineBasicBlock* NextMBB,
2676  BranchProbability BranchProbToNext,
2677  unsigned Reg,
2678  BitTestCase &B,
2679  MachineBasicBlock *SwitchBB) {
2680  SDLoc dl = getCurSDLoc();
2681  MVT VT = BB.RegVT;
2682  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2683  SDValue Cmp;
2684  unsigned PopCount = countPopulation(B.Mask);
2685  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2686  if (PopCount == 1) {
2687  // Testing for a single bit; just compare the shift count with what it
2688  // would need to be to shift a 1 bit in that position.
2689  Cmp = DAG.getSetCC(
2690  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2691  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2692  ISD::SETEQ);
2693  } else if (PopCount == BB.Range) {
2694  // There is only one zero bit in the range, test for it directly.
2695  Cmp = DAG.getSetCC(
2696  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2697  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2698  ISD::SETNE);
2699  } else {
2700  // Make desired shift
2701  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2702  DAG.getConstant(1, dl, VT), ShiftOp);
2703 
2704  // Emit bit tests and jumps
2705  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2706  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2707  Cmp = DAG.getSetCC(
2708  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2709  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2710  }
2711 
2712  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2713  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2714  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2715  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2716  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2717  // one as they are relative probabilities (and thus work more like weights),
2718  // and hence we need to normalize them to let the sum of them become one.
2719  SwitchBB->normalizeSuccProbs();
2720 
2721  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2722  MVT::Other, getControlRoot(),
2723  Cmp, DAG.getBasicBlock(B.TargetBB));
2724 
2725  // Avoid emitting unnecessary branches to the next block.
2726  if (NextMBB != NextBlock(SwitchBB))
2727  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2728  DAG.getBasicBlock(NextMBB));
2729 
2730  DAG.setRoot(BrAnd);
2731 }
2732 
2733 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2734  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2735 
2736  // Retrieve successors. Look through artificial IR level blocks like
2737  // catchswitch for successors.
2738  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2739  const BasicBlock *EHPadBB = I.getSuccessor(1);
2740 
2741  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2742  // have to do anything here to lower funclet bundles.
2744  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2745  "Cannot lower invokes with arbitrary operand bundles yet!");
2746 
2747  const Value *Callee(I.getCalledValue());
2748  const Function *Fn = dyn_cast<Function>(Callee);
2749  if (isa<InlineAsm>(Callee))
2750  visitInlineAsm(&I);
2751  else if (Fn && Fn->isIntrinsic()) {
2752  switch (Fn->getIntrinsicID()) {
2753  default:
2754  llvm_unreachable("Cannot invoke this intrinsic");
2755  case Intrinsic::donothing:
2756  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2757  break;
2758  case Intrinsic::experimental_patchpoint_void:
2759  case Intrinsic::experimental_patchpoint_i64:
2760  visitPatchpoint(&I, EHPadBB);
2761  break;
2762  case Intrinsic::experimental_gc_statepoint:
2763  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2764  break;
2765  case Intrinsic::wasm_rethrow_in_catch: {
2766  // This is usually done in visitTargetIntrinsic, but this intrinsic is
2767  // special because it can be invoked, so we manually lower it to a DAG
2768  // node here.
2770  Ops.push_back(getRoot()); // inchain
2771  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2772  Ops.push_back(
2773  DAG.getTargetConstant(Intrinsic::wasm_rethrow_in_catch, getCurSDLoc(),
2774  TLI.getPointerTy(DAG.getDataLayout())));
2775  SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2776  DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2777  break;
2778  }
2779  }
2781  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2782  // Eventually we will support lowering the @llvm.experimental.deoptimize
2783  // intrinsic, and right now there are no plans to support other intrinsics
2784  // with deopt state.
2785  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2786  } else {
2787  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2788  }
2789 
2790  // If the value of the invoke is used outside of its defining block, make it
2791  // available as a virtual register.
2792  // We already took care of the exported value for the statepoint instruction
2793  // during call to the LowerStatepoint.
2794  if (!isStatepoint(I)) {
2795  CopyToExportRegsIfNeeded(&I);
2796  }
2797 
2799  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2800  BranchProbability EHPadBBProb =
2801  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2803  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2804 
2805  // Update successor info.
2806  addSuccessorWithProb(InvokeMBB, Return);
2807  for (auto &UnwindDest : UnwindDests) {
2808  UnwindDest.first->setIsEHPad();
2809  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2810  }
2811  InvokeMBB->normalizeSuccProbs();
2812 
2813  // Drop into normal successor.
2814  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2815  DAG.getBasicBlock(Return)));
2816 }
2817 
2818 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2819  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2820 
2821  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2822  // have to do anything here to lower funclet bundles.
2824  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2825  "Cannot lower callbrs with arbitrary operand bundles yet!");
2826 
2827  assert(isa<InlineAsm>(I.getCalledValue()) &&
2828  "Only know how to handle inlineasm callbr");
2829  visitInlineAsm(&I);
2830 
2831  // Retrieve successors.
2832  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2833 
2834  // Update successor info.
2835  addSuccessorWithProb(CallBrMBB, Return);
2836  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2837  MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2838  addSuccessorWithProb(CallBrMBB, Target);
2839  }
2840  CallBrMBB->normalizeSuccProbs();
2841 
2842  // Drop into default successor.
2843  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2844  MVT::Other, getControlRoot(),
2845  DAG.getBasicBlock(Return)));
2846 }
2847 
2848 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2849  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2850 }
2851 
2852 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2853  assert(FuncInfo.MBB->isEHPad() &&
2854  "Call to landingpad not in landing pad!");
2855 
2856  // If there aren't registers to copy the values into (e.g., during SjLj
2857  // exceptions), then don't bother to create these DAG nodes.
2858  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2859  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2860  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2861  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2862  return;
2863 
2864  // If landingpad's return type is token type, we don't create DAG nodes
2865  // for its exception pointer and selector value. The extraction of exception
2866  // pointer or selector value from token type landingpads is not currently
2867  // supported.
2868  if (LP.getType()->isTokenTy())
2869  return;
2870 
2872  SDLoc dl = getCurSDLoc();
2873  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2874  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2875 
2876  // Get the two live-in registers as SDValues. The physregs have already been
2877  // copied into virtual registers.
2878  SDValue Ops[2];
2879  if (FuncInfo.ExceptionPointerVirtReg) {
2880  Ops[0] = DAG.getZExtOrTrunc(
2881  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2882  FuncInfo.ExceptionPointerVirtReg,
2883  TLI.getPointerTy(DAG.getDataLayout())),
2884  dl, ValueVTs[0]);
2885  } else {
2886  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2887  }
2888  Ops[1] = DAG.getZExtOrTrunc(
2889  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2890  FuncInfo.ExceptionSelectorVirtReg,
2891  TLI.getPointerTy(DAG.getDataLayout())),
2892  dl, ValueVTs[1]);
2893 
2894  // Merge into one.
2895  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2896  DAG.getVTList(ValueVTs), Ops);
2897  setValue(&LP, Res);
2898 }
2899 
2901  MachineBasicBlock *Last) {
2902  // Update JTCases.
2903  for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
2904  if (SL->JTCases[i].first.HeaderBB == First)
2905  SL->JTCases[i].first.HeaderBB = Last;
2906 
2907  // Update BitTestCases.
2908  for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
2909  if (SL->BitTestCases[i].Parent == First)
2910  SL->BitTestCases[i].Parent = Last;
2911 }
2912 
2913 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2914  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2915 
2916  // Update machine-CFG edges with unique successors.
2918  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2919  BasicBlock *BB = I.getSuccessor(i);
2920  bool Inserted = Done.insert(BB).second;
2921  if (!Inserted)
2922  continue;
2923 
2924  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2925  addSuccessorWithProb(IndirectBrMBB, Succ);
2926  }
2927  IndirectBrMBB->normalizeSuccProbs();
2928 
2929  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2930  MVT::Other, getControlRoot(),
2931  getValue(I.getAddress())));
2932 }
2933 
2934 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2935  if (!DAG.getTarget().Options.TrapUnreachable)
2936  return;
2937 
2938  // We may be able to ignore unreachable behind a noreturn call.
2940  const BasicBlock &BB = *I.getParent();
2941  if (&I != &BB.front()) {
2943  std::prev(BasicBlock::const_iterator(&I));
2944  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2945  if (Call->doesNotReturn())
2946  return;
2947  }
2948  }
2949  }
2950 
2951  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2952 }
2953 
2954 void SelectionDAGBuilder::visitFSub(const User &I) {
2955  // -0.0 - X --> fneg
2956  Type *Ty = I.getType();
2957  if (isa<Constant>(I.getOperand(0)) &&
2959  SDValue Op2 = getValue(I.getOperand(1));
2960  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2961  Op2.getValueType(), Op2));
2962  return;
2963  }
2964 
2965  visitBinary(I, ISD::FSUB);
2966 }
2967 
2968 /// Checks if the given instruction performs a vector reduction, in which case
2969 /// we have the freedom to alter the elements in the result as long as the
2970 /// reduction of them stays unchanged.
2971 static bool isVectorReductionOp(const User *I) {
2972  const Instruction *Inst = dyn_cast<Instruction>(I);
2973  if (!Inst || !Inst->getType()->isVectorTy())
2974  return false;
2975 
2976  auto OpCode = Inst->getOpcode();
2977  switch (OpCode) {
2978  case Instruction::Add:
2979  case Instruction::Mul:
2980  case Instruction::And:
2981  case Instruction::Or:
2982  case Instruction::Xor:
2983  break;
2984  case Instruction::FAdd:
2985  case Instruction::FMul:
2986  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2987  if (FPOp->getFastMathFlags().isFast())
2988  break;
2990  default:
2991  return false;
2992  }
2993 
2994  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2995  // Ensure the reduction size is a power of 2.
2996  if (!isPowerOf2_32(ElemNum))
2997  return false;
2998 
2999  unsigned ElemNumToReduce = ElemNum;
3000 
3001  // Do DFS search on the def-use chain from the given instruction. We only
3002  // allow four kinds of operations during the search until we reach the
3003  // instruction that extracts the first element from the vector:
3004  //
3005  // 1. The reduction operation of the same opcode as the given instruction.
3006  //
3007  // 2. PHI node.
3008  //
3009  // 3. ShuffleVector instruction together with a reduction operation that
3010  // does a partial reduction.
3011  //
3012  // 4. ExtractElement that extracts the first element from the vector, and we
3013  // stop searching the def-use chain here.
3014  //
3015  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
3016  // from 1-3 to the stack to continue the DFS. The given instruction is not
3017  // a reduction operation if we meet any other instructions other than those
3018  // listed above.
3019 
3020  SmallVector<const User *, 16> UsersToVisit{Inst};
3022  bool ReduxExtracted = false;
3023 
3024  while (!UsersToVisit.empty()) {
3025  auto User = UsersToVisit.back();
3026  UsersToVisit.pop_back();
3027  if (!Visited.insert(User).second)
3028  continue;
3029 
3030  for (const auto &U : User->users()) {
3031  auto Inst = dyn_cast<Instruction>(U);
3032  if (!Inst)
3033  return false;
3034 
3035  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
3036  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
3037  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
3038  return false;
3039  UsersToVisit.push_back(U);
3040  } else if (const ShuffleVectorInst *ShufInst =
3041  dyn_cast<ShuffleVectorInst>(U)) {
3042  // Detect the following pattern: A ShuffleVector instruction together
3043  // with a reduction that do partial reduction on the first and second
3044  // ElemNumToReduce / 2 elements, and store the result in
3045  // ElemNumToReduce / 2 elements in another vector.
3046 
3047  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
3048  if (ResultElements < ElemNum)
3049  return false;
3050 
3051  if (ElemNumToReduce == 1)
3052  return false;
3053  if (!isa<UndefValue>(U->getOperand(1)))
3054  return false;
3055  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
3056  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
3057  return false;
3058  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
3059  if (ShufInst->getMaskValue(i) != -1)
3060  return false;
3061 
3062  // There is only one user of this ShuffleVector instruction, which
3063  // must be a reduction operation.
3064  if (!U->hasOneUse())
3065  return false;
3066 
3067  auto U2 = dyn_cast<Instruction>(*U->user_begin());
3068  if (!U2 || U2->getOpcode() != OpCode)
3069  return false;
3070 
3071  // Check operands of the reduction operation.
3072  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
3073  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
3074  UsersToVisit.push_back(U2);
3075  ElemNumToReduce /= 2;
3076  } else
3077  return false;
3078  } else if (isa<ExtractElementInst>(U)) {
3079  // At this moment we should have reduced all elements in the vector.
3080  if (ElemNumToReduce != 1)
3081  return false;
3082 
3083  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
3084  if (!Val || !Val->isZero())
3085  return false;
3086 
3087  ReduxExtracted = true;
3088  } else
3089  return false;
3090  }
3091  }
3092  return ReduxExtracted;
3093 }
3094 
3095 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3096  SDNodeFlags Flags;
3097 
3098  SDValue Op = getValue(I.getOperand(0));
3099  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3100  Op, Flags);
3101  setValue(&I, UnNodeValue);
3102 }
3103 
3104 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3105  SDNodeFlags Flags;
3106  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3107  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3108  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3109  }
3110  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
3111  Flags.setExact(ExactOp->isExact());
3112  }
3113  if (isVectorReductionOp(&I)) {
3114  Flags.setVectorReduction(true);
3115  LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
3116  }
3117 
3118  SDValue Op1 = getValue(I.getOperand(0));
3119  SDValue Op2 = getValue(I.getOperand(1));
3120  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3121  Op1, Op2, Flags);
3122  setValue(&I, BinNodeValue);
3123 }
3124 
3125 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3126  SDValue Op1 = getValue(I.getOperand(0));
3127  SDValue Op2 = getValue(I.getOperand(1));
3128 
3129  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3130  Op1.getValueType(), DAG.getDataLayout());
3131 
3132  // Coerce the shift amount to the right type if we can.
3133  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3134  unsigned ShiftSize = ShiftTy.getSizeInBits();
3135  unsigned Op2Size = Op2.getValueSizeInBits();
3136  SDLoc DL = getCurSDLoc();
3137 
3138  // If the operand is smaller than the shift count type, promote it.
3139  if (ShiftSize > Op2Size)
3140  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3141 
3142  // If the operand is larger than the shift count type but the shift
3143  // count type has enough bits to represent any shift value, truncate
3144  // it now. This is a common case and it exposes the truncate to
3145  // optimization early.
3146  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3147  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3148  // Otherwise we'll need to temporarily settle for some other convenient
3149  // type. Type legalization will make adjustments once the shiftee is split.
3150  else
3151  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3152  }
3153 
3154  bool nuw = false;
3155  bool nsw = false;
3156  bool exact = false;
3157 
3158  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3159 
3160  if (const OverflowingBinaryOperator *OFBinOp =
3161  dyn_cast<const OverflowingBinaryOperator>(&I)) {
3162  nuw = OFBinOp->hasNoUnsignedWrap();
3163  nsw = OFBinOp->hasNoSignedWrap();
3164  }
3165  if (const PossiblyExactOperator *ExactOp =
3166  dyn_cast<const PossiblyExactOperator>(&I))
3167  exact = ExactOp->isExact();
3168  }
3169  SDNodeFlags Flags;
3170  Flags.setExact(exact);
3171  Flags.setNoSignedWrap(nsw);
3172  Flags.setNoUnsignedWrap(nuw);
3173  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3174  Flags);
3175  setValue(&I, Res);
3176 }
3177 
3178 void SelectionDAGBuilder::visitSDiv(const User &I) {
3179  SDValue Op1 = getValue(I.getOperand(0));
3180  SDValue Op2 = getValue(I.getOperand(1));
3181 
3182  SDNodeFlags Flags;
3183  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3184  cast<PossiblyExactOperator>(&I)->isExact());
3185  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3186  Op2, Flags));
3187 }
3188 
3189 void SelectionDAGBuilder::visitICmp(const User &I) {
3191  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3192  predicate = IC->getPredicate();
3193  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3194  predicate = ICmpInst::Predicate(IC->getPredicate());
3195  SDValue Op1 = getValue(I.getOperand(0));
3196  SDValue Op2 = getValue(I.getOperand(1));
3197  ISD::CondCode Opcode = getICmpCondCode(predicate);
3198 
3199  auto &TLI = DAG.getTargetLoweringInfo();
3200  EVT MemVT =
3201  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3202 
3203  // If a pointer's DAG type is larger than its memory type then the DAG values
3204  // are zero-extended. This breaks signed comparisons so truncate back to the
3205  // underlying type before doing the compare.
3206  if (Op1.getValueType() != MemVT) {
3207  Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3208  Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3209  }
3210 
3211  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3212  I.getType());
3213  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3214 }
3215 
3216 void SelectionDAGBuilder::visitFCmp(const User &I) {
3218  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3219  predicate = FC->getPredicate();
3220  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3221  predicate = FCmpInst::Predicate(FC->getPredicate());
3222  SDValue Op1 = getValue(I.getOperand(0));
3223  SDValue Op2 = getValue(I.getOperand(1));
3224 
3225  ISD::CondCode Condition = getFCmpCondCode(predicate);
3226  auto *FPMO = dyn_cast<FPMathOperator>(&I);
3227  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
3228  Condition = getFCmpCodeWithoutNaN(Condition);
3229 
3230  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3231  I.getType());
3232  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3233 }
3234 
3235 // Check if the condition of the select has one use or two users that are both
3236 // selects with the same condition.
3237 static bool hasOnlySelectUsers(const Value *Cond) {
3238  return llvm::all_of(Cond->users(), [](const Value *V) {
3239  return isa<SelectInst>(V);
3240  });
3241 }
3242 
3243 void SelectionDAGBuilder::visitSelect(const User &I) {
3246  ValueVTs);
3247  unsigned NumValues = ValueVTs.size();
3248  if (NumValues == 0) return;
3249 
3250  SmallVector<SDValue, 4> Values(NumValues);
3251  SDValue Cond = getValue(I.getOperand(0));
3252  SDValue LHSVal = getValue(I.getOperand(1));
3253  SDValue RHSVal = getValue(I.getOperand(2));
3254  auto BaseOps = {Cond};
3255  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
3257 
3258  bool IsUnaryAbs = false;
3259 
3260  // Min/max matching is only viable if all output VTs are the same.
3261  if (is_splat(ValueVTs)) {
3262  EVT VT = ValueVTs[0];
3263  LLVMContext &Ctx = *DAG.getContext();
3264  auto &TLI = DAG.getTargetLoweringInfo();
3265 
3266  // We care about the legality of the operation after it has been type
3267  // legalized.
3268  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
3269  VT != TLI.getTypeToTransformTo(Ctx, VT))
3270  VT = TLI.getTypeToTransformTo(Ctx, VT);
3271 
3272  // If the vselect is legal, assume we want to leave this as a vector setcc +
3273  // vselect. Otherwise, if this is going to be scalarized, we want to see if
3274  // min/max is legal on the scalar type.
3275  bool UseScalarMinMax = VT.isVector() &&
3276  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3277 
3278  Value *LHS, *RHS;
3279  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3281  switch (SPR.Flavor) {
3282  case SPF_UMAX: Opc = ISD::UMAX; break;
3283  case SPF_UMIN: Opc = ISD::UMIN; break;
3284  case SPF_SMAX: Opc = ISD::SMAX; break;
3285  case SPF_SMIN: Opc = ISD::SMIN; break;
3286  case SPF_FMINNUM:
3287  switch (SPR.NaNBehavior) {
3288  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3289  case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3290  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3291  case SPNB_RETURNS_ANY: {
3292  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3293  Opc = ISD::FMINNUM;
3294  else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3295  Opc = ISD::FMINIMUM;
3296  else if (UseScalarMinMax)
3297  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3299  break;
3300  }
3301  }
3302  break;
3303  case SPF_FMAXNUM:
3304  switch (SPR.NaNBehavior) {
3305  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3306  case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3307  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3308  case SPNB_RETURNS_ANY:
3309 
3310  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3311  Opc = ISD::FMAXNUM;
3312  else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3313  Opc = ISD::FMAXIMUM;
3314  else if (UseScalarMinMax)
3315  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3317  break;
3318  }
3319  break;
3320  case SPF_ABS:
3321  IsUnaryAbs = true;
3322  Opc = ISD::ABS;
3323  break;
3324  case SPF_NABS:
3325  // TODO: we need to produce sub(0, abs(X)).
3326  default: break;
3327  }
3328 
3329  if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3330  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3331  (UseScalarMinMax &&
3332  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3333  // If the underlying comparison instruction is used by any other
3334  // instruction, the consumed instructions won't be destroyed, so it is
3335  // not profitable to convert to a min/max.
3336  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3337  OpCode = Opc;
3338  LHSVal = getValue(LHS);
3339  RHSVal = getValue(RHS);
3340  BaseOps = {};
3341  }
3342 
3343  if (IsUnaryAbs) {
3344  OpCode = Opc;
3345  LHSVal = getValue(LHS);
3346  BaseOps = {};
3347  }
3348  }
3349 
3350  if (IsUnaryAbs) {
3351  for (unsigned i = 0; i != NumValues; ++i) {
3352  Values[i] =
3353  DAG.getNode(OpCode, getCurSDLoc(),
3354  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),
3355  SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3356  }
3357  } else {
3358  for (unsigned i = 0; i != NumValues; ++i) {
3359  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3360  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3361  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3362  Values[i] = DAG.getNode(
3363  OpCode, getCurSDLoc(),
3364  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
3365  }
3366  }
3367 
3368  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3369  DAG.getVTList(ValueVTs), Values));
3370 }
3371 
3372 void SelectionDAGBuilder::visitTrunc(const User &I) {
3373  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3374  SDValue N = getValue(I.getOperand(0));
3375  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3376  I.getType());
3377  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3378 }
3379 
3380 void SelectionDAGBuilder::visitZExt(const User &I) {
3381  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3382  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3383  SDValue N = getValue(I.getOperand(0));
3384  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3385  I.getType());
3386  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3387 }
3388 
3389 void SelectionDAGBuilder::visitSExt(const User &I) {
3390  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3391  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3392  SDValue N = getValue(I.getOperand(0));
3393  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3394  I.getType());
3395  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3396 }
3397 
3398 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3399  // FPTrunc is never a no-op cast, no need to check
3400  SDValue N = getValue(I.getOperand(0));
3401  SDLoc dl = getCurSDLoc();
3402  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3403  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3404  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3405  DAG.getTargetConstant(
3406  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3407 }
3408 
3409 void SelectionDAGBuilder::visitFPExt(const User &I) {
3410  // FPExt is never a no-op cast, no need to check
3411  SDValue N = getValue(I.getOperand(0));
3412  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3413  I.getType());
3414  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3415 }
3416 
3417 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3418  // FPToUI is never a no-op cast, no need to check
3419  SDValue N = getValue(I.getOperand(0));
3420  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3421  I.getType());
3422  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3423 }
3424 
3425 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3426  // FPToSI is never a no-op cast, no need to check
3427  SDValue N = getValue(I.getOperand(0));
3428  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3429  I.getType());
3430  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3431 }
3432 
3433 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3434  // UIToFP is never a no-op cast, no need to check
3435  SDValue N = getValue(I.getOperand(0));
3436  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3437  I.getType());
3438  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3439 }
3440 
3441 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3442  // SIToFP is never a no-op cast, no need to check
3443  SDValue N = getValue(I.getOperand(0));
3444  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3445  I.getType());
3446  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3447 }
3448 
3449 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3450  // What to do depends on the size of the integer and the size of the pointer.
3451  // We can either truncate, zero extend, or no-op, accordingly.
3452  SDValue N = getValue(I.getOperand(0));
3453  auto &TLI = DAG.getTargetLoweringInfo();
3454  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3455  I.getType());
3456  EVT PtrMemVT =
3457  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3458  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3459  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3460  setValue(&I, N);
3461 }
3462 
3463 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3464  // What to do depends on the size of the integer and the size of the pointer.
3465  // We can either truncate, zero extend, or no-op, accordingly.
3466  SDValue N = getValue(I.getOperand(0));
3467  auto &TLI = DAG.getTargetLoweringInfo();
3468  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3469  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3470  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3471  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3472  setValue(&I, N);
3473 }
3474 
3475 void SelectionDAGBuilder::visitBitCast(const User &I) {
3476  SDValue N = getValue(I.getOperand(0));
3477  SDLoc dl = getCurSDLoc();
3478  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3479  I.getType());
3480 
3481  // BitCast assures us that source and destination are the same size so this is
3482  // either a BITCAST or a no-op.
3483  if (DestVT != N.getValueType())
3484  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3485  DestVT, N)); // convert types.
3486  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3487  // might fold any kind of constant expression to an integer constant and that
3488  // is not what we are looking for. Only recognize a bitcast of a genuine
3489  // constant integer as an opaque constant.
3490  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3491  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3492  /*isOpaque*/true));
3493  else
3494  setValue(&I, N); // noop cast.
3495 }
3496 
3497 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3498  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3499  const Value *SV = I.getOperand(0);
3500  SDValue N = getValue(SV);
3501  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3502 
3503  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3504  unsigned DestAS = I.getType()->getPointerAddressSpace();
3505 
3506  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3507  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3508 
3509  setValue(&I, N);
3510 }
3511 
3512 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3513  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3514  SDValue InVec = getValue(I.getOperand(0));
3515  SDValue InVal = getValue(I.getOperand(1));
3516  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3517  TLI.getVectorIdxTy(DAG.getDataLayout()));
3518  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3519  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3520  InVec, InVal, InIdx));
3521 }
3522 
3523 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3524  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3525  SDValue InVec = getValue(I.getOperand(0));
3526  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3527  TLI.getVectorIdxTy(DAG.getDataLayout()));
3528  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3529  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3530  InVec, InIdx));
3531 }
3532 
3533 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3534  SDValue Src1 = getValue(I.getOperand(0));
3535  SDValue Src2 = getValue(I.getOperand(1));
3536  SDLoc DL = getCurSDLoc();
3537 
3539  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3540  unsigned MaskNumElts = Mask.size();
3541 
3542  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3543  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3544  EVT SrcVT = Src1.getValueType();
3545  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3546 
3547  if (SrcNumElts == MaskNumElts) {
3548  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3549  return;
3550  }
3551 
3552  // Normalize the shuffle vector since mask and vector length don't match.
3553  if (SrcNumElts < MaskNumElts) {
3554  // Mask is longer than the source vectors. We can use concatenate vector to
3555  // make the mask and vectors lengths match.
3556 
3557  if (MaskNumElts % SrcNumElts == 0) {
3558  // Mask length is a multiple of the source vector length.
3559  // Check if the shuffle is some kind of concatenation of the input
3560  // vectors.
3561  unsigned NumConcat = MaskNumElts / SrcNumElts;
3562  bool IsConcat = true;
3563  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3564  for (unsigned i = 0; i != MaskNumElts; ++i) {
3565  int Idx = Mask[i];
3566  if (Idx < 0)
3567  continue;
3568  // Ensure the indices in each SrcVT sized piece are sequential and that
3569  // the same source is used for the whole piece.
3570  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3571  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3572  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3573  IsConcat = false;
3574  break;
3575  }
3576  // Remember which source this index came from.
3577  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3578  }
3579 
3580  // The shuffle is concatenating multiple vectors together. Just emit
3581  // a CONCAT_VECTORS operation.
3582  if (IsConcat) {
3583  SmallVector<SDValue, 8> ConcatOps;
3584  for (auto Src : ConcatSrcs) {
3585  if (Src < 0)
3586  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3587  else if (Src == 0)
3588  ConcatOps.push_back(Src1);
3589  else
3590  ConcatOps.push_back(Src2);
3591  }
3592  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3593  return;
3594  }
3595  }
3596 
3597  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3598  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3599  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3600  PaddedMaskNumElts);
3601 
3602  // Pad both vectors with undefs to make them the same length as the mask.
3603  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3604 
3605  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3606  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3607  MOps1[0] = Src1;
3608  MOps2[0] = Src2;
3609 
3610  Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3611  Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3612 
3613  // Readjust mask for new input vector length.
3614  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3615  for (unsigned i = 0; i != MaskNumElts; ++i) {
3616  int Idx = Mask[i];
3617  if (Idx >= (int)SrcNumElts)
3618  Idx -= SrcNumElts - PaddedMaskNumElts;
3619  MappedOps[i] = Idx;
3620  }
3621 
3622  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3623 
3624  // If the concatenated vector was padded, extract a subvector with the
3625  // correct number of elements.
3626  if (MaskNumElts != PaddedMaskNumElts)
3627  Result = DAG.getNode(
3628  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3629  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3630 
3631  setValue(&I, Result);
3632  return;
3633  }
3634 
3635  if (SrcNumElts > MaskNumElts) {
3636  // Analyze the access pattern of the vector to see if we can extract
3637  // two subvectors and do the shuffle.
3638  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3639  bool CanExtract = true;
3640  for (int Idx : Mask) {
3641  unsigned Input = 0;
3642  if (Idx < 0)
3643  continue;
3644 
3645  if (Idx >= (int)SrcNumElts) {
3646  Input = 1;
3647  Idx -= SrcNumElts;
3648  }
3649 
3650  // If all the indices come from the same MaskNumElts sized portion of
3651  // the sources we can use extract. Also make sure the extract wouldn't
3652  // extract past the end of the source.
3653  int NewStartIdx = alignDown(Idx, MaskNumElts);
3654  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3655  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3656  CanExtract = false;
3657  // Make sure we always update StartIdx as we use it to track if all
3658  // elements are undef.
3659  StartIdx[Input] = NewStartIdx;
3660  }
3661 
3662  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3663  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3664  return;
3665  }
3666  if (CanExtract) {
3667  // Extract appropriate subvector and generate a vector shuffle
3668  for (unsigned Input = 0; Input < 2; ++Input) {
3669  SDValue &Src = Input == 0 ? Src1 : Src2;
3670  if (StartIdx[Input] < 0)
3671  Src = DAG.getUNDEF(VT);
3672  else {
3673  Src = DAG.getNode(
3674  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3675  DAG.getConstant(StartIdx[Input], DL,
3676  TLI.getVectorIdxTy(DAG.getDataLayout())));
3677  }
3678  }
3679 
3680  // Calculate new mask.
3681  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3682  for (int &Idx : MappedOps) {
3683  if (Idx >= (int)SrcNumElts)
3684  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3685  else if (Idx >= 0)
3686  Idx -= StartIdx[0];
3687  }
3688 
3689  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3690  return;
3691  }
3692  }
3693 
3694  // We can't use either concat vectors or extract subvectors so fall back to
3695  // replacing the shuffle with extract and build vector.
3696  // to insert and build vector.
3697  EVT EltVT = VT.getVectorElementType();
3698  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3700  for (int Idx : Mask) {
3701  SDValue Res;
3702 
3703  if (Idx < 0) {
3704  Res = DAG.getUNDEF(EltVT);
3705  } else {
3706  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3707  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3708 
3709  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3710  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3711  }
3712 
3713  Ops.push_back(Res);
3714  }
3715 
3716  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3717 }
3718 
3719 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3720  ArrayRef<unsigned> Indices;
3721  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3722  Indices = IV->getIndices();
3723  else
3724  Indices = cast<ConstantExpr>(&I)->getIndices();
3725 
3726  const Value *Op0 = I.getOperand(0);
3727  const Value *Op1 = I.getOperand(1);
3728  Type *AggTy = I.getType();
3729  Type *ValTy = Op1->getType();
3730  bool IntoUndef = isa<UndefValue>(Op0);
3731  bool FromUndef = isa<UndefValue>(Op1);
3732 
3733  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3734 
3735  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3736  SmallVector<EVT, 4> AggValueVTs;
3737  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3738  SmallVector<EVT, 4> ValValueVTs;
3739  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3740 
3741  unsigned NumAggValues = AggValueVTs.size();
3742  unsigned NumValValues = ValValueVTs.size();
3743  SmallVector<SDValue, 4> Values(NumAggValues);
3744 
3745  // Ignore an insertvalue that produces an empty object
3746  if (!NumAggValues) {
3747  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3748  return;
3749  }
3750 
3751  SDValue Agg = getValue(Op0);
3752  unsigned i = 0;
3753  // Copy the beginning value(s) from the original aggregate.
3754  for (; i != LinearIndex; ++i)
3755  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3756  SDValue(Agg.getNode(), Agg.getResNo() + i);
3757  // Copy values from the inserted value(s).
3758  if (NumValValues) {
3759  SDValue Val = getValue(Op1);
3760  for (; i != LinearIndex + NumValValues; ++i)
3761  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3762  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3763  }
3764  // Copy remaining value(s) from the original aggregate.
3765  for (; i != NumAggValues; ++i)
3766  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3767  SDValue(Agg.getNode(), Agg.getResNo() + i);
3768 
3769  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3770  DAG.getVTList(AggValueVTs), Values));
3771 }
3772 
3773 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3774  ArrayRef<unsigned> Indices;
3775  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3776  Indices = EV->getIndices();
3777  else
3778  Indices = cast<ConstantExpr>(&I)->getIndices();
3779 
3780  const Value *Op0 = I.getOperand(0);
3781  Type *AggTy = Op0->getType();
3782  Type *ValTy = I.getType();
3783  bool OutOfUndef = isa<UndefValue>(Op0);
3784 
3785  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3786 
3787  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3788  SmallVector<EVT, 4> ValValueVTs;
3789  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3790 
3791  unsigned NumValValues = ValValueVTs.size();
3792 
3793  // Ignore a extractvalue that produces an empty object
3794  if (!NumValValues) {
3795  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3796  return;
3797  }
3798 
3799  SmallVector<SDValue, 4> Values(NumValValues);
3800 
3801  SDValue Agg = getValue(Op0);
3802  // Copy out the selected value(s).
3803  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3804  Values[i - LinearIndex] =
3805  OutOfUndef ?
3806  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3807  SDValue(Agg.getNode(), Agg.getResNo() + i);
3808 
3809  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3810  DAG.getVTList(ValValueVTs), Values));
3811 }
3812 
3813 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3814  Value *Op0 = I.getOperand(0);
3815  // Note that the pointer operand may be a vector of pointers. Take the scalar
3816  // element which holds a pointer.
3817  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3818  SDValue N = getValue(Op0);
3819  SDLoc dl = getCurSDLoc();
3820  auto &TLI = DAG.getTargetLoweringInfo();
3821  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3822  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3823 
3824  // Normalize Vector GEP - all scalar operands should be converted to the
3825  // splat vector.
3826  unsigned VectorWidth = I.getType()->isVectorTy() ?
3827  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3828 
3829  if (VectorWidth && !N.getValueType().isVector()) {
3830  LLVMContext &Context = *DAG.getContext();
3831  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3832  N = DAG.getSplatBuildVector(VT, dl, N);
3833  }
3834 
3835  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3836  GTI != E; ++GTI) {
3837  const Value *Idx = GTI.getOperand();
3838  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3839  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3840  if (Field) {
3841  // N = N + Offset
3842  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3843 
3844  // In an inbounds GEP with an offset that is nonnegative even when
3845  // interpreted as signed, assume there is no unsigned overflow.
3846  SDNodeFlags Flags;
3847  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3848  Flags.setNoUnsignedWrap(true);
3849 
3850  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3851  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3852  }
3853  } else {
3854  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3855  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3856  APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3857 
3858  // If this is a scalar constant or a splat vector of constants,
3859  // handle it quickly.
3860  const auto *CI = dyn_cast<ConstantInt>(Idx);
3861  if (!CI && isa<ConstantDataVector>(Idx) &&
3862  cast<ConstantDataVector>(Idx)->getSplatValue())
3863  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3864 
3865  if (CI) {
3866  if (CI->isZero())
3867  continue;
3868  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3869  LLVMContext &Context = *DAG.getContext();
3870  SDValue OffsVal = VectorWidth ?
3871  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3872  DAG.getConstant(Offs, dl, IdxTy);
3873 
3874  // In an inbouds GEP with an offset that is nonnegative even when
3875  // interpreted as signed, assume there is no unsigned overflow.
3876  SDNodeFlags Flags;
3877  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3878  Flags.setNoUnsignedWrap(true);
3879 
3880  OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3881 
3882  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3883  continue;
3884  }
3885 
3886  // N = N + Idx * ElementSize;
3887  SDValue IdxN = getValue(Idx);
3888 
3889  if (!IdxN.getValueType().isVector() && VectorWidth) {
3890  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3891  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3892  }
3893 
3894  // If the index is smaller or larger than intptr_t, truncate or extend
3895  // it.
3896  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3897 
3898  // If this is a multiply by a power of two, turn it into a shl
3899  // immediately. This is a very common case.
3900  if (ElementSize != 1) {
3901  if (ElementSize.isPowerOf2()) {
3902  unsigned Amt = ElementSize.logBase2();
3903  IdxN = DAG.getNode(ISD::SHL, dl,
3904  N.getValueType(), IdxN,
3905  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3906  } else {
3907  SDValue Scale = DAG.getConstant(ElementSize.getZExtValue(), dl,
3908  IdxN.getValueType());
3909  IdxN = DAG.getNode(ISD::MUL, dl,
3910  N.getValueType(), IdxN, Scale);
3911  }
3912  }
3913 
3914  N = DAG.getNode(ISD::ADD, dl,
3915  N.getValueType(), N, IdxN);
3916  }
3917  }
3918 
3919  if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3920  N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3921 
3922  setValue(&I, N);
3923 }
3924 
3925 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3926  // If this is a fixed sized alloca in the entry block of the function,
3927  // allocate it statically on the stack.
3928  if (FuncInfo.StaticAllocaMap.count(&I))
3929  return; // getValue will auto-populate this.
3930 
3931  SDLoc dl = getCurSDLoc();
3932  Type *Ty = I.getAllocatedType();
3933  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3934  auto &DL = DAG.getDataLayout();
3935  uint64_t TySize = DL.getTypeAllocSize(Ty);
3936  unsigned Align =
3937  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3938 
3939  SDValue AllocSize = getValue(I.getArraySize());
3940 
3941  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3942  if (AllocSize.getValueType() != IntPtr)
3943  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3944 
3945  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3946  AllocSize,
3947  DAG.getConstant(TySize, dl, IntPtr));
3948 
3949  // Handle alignment. If the requested alignment is less than or equal to
3950  // the stack alignment, ignore it. If the size is greater than or equal to
3951  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3952  unsigned StackAlign =
3954  if (Align <= StackAlign)
3955  Align = 0;
3956 
3957  // Round the size of the allocation up to the stack alignment size
3958  // by add SA-1 to the size. This doesn't overflow because we're computing
3959  // an address inside an alloca.
3960  SDNodeFlags Flags;
3961  Flags.setNoUnsignedWrap(true);
3962  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3963  DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3964 
3965  // Mask out the low bits for alignment purposes.
3966  AllocSize =
3967  DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3968  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3969 
3970  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3971  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3972  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3973  setValue(&I, DSA);
3974  DAG.setRoot(DSA.getValue(1));
3975 
3976  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3977 }
3978 
3979 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3980  if (I.isAtomic())
3981  return visitAtomicLoad(I);
3982 
3983  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3984  const Value *SV = I.getOperand(0);
3985  if (TLI.supportSwiftError()) {
3986  // Swifterror values can come from either a function parameter with
3987  // swifterror attribute or an alloca with swifterror attribute.
3988  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3989  if (Arg->hasSwiftErrorAttr())
3990  return visitLoadFromSwiftError(I);
3991  }
3992 
3993  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3994  if (Alloca->isSwiftError())
3995  return visitLoadFromSwiftError(I);
3996  }
3997  }
3998 
3999  SDValue Ptr = getValue(SV);
4000 
4001  Type *Ty = I.getType();
4002 
4003  bool isVolatile = I.isVolatile();
4004  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
4005  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
4006  bool isDereferenceable =
4008  unsigned Alignment = I.getAlignment();
4009 
4010  AAMDNodes AAInfo;
4011  I.getAAMetadata(AAInfo);
4012  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4013 
4014  SmallVector<EVT, 4> ValueVTs, MemVTs;
4016  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4017  unsigned NumValues = ValueVTs.size();
4018  if (NumValues == 0)
4019  return;
4020 
4021  SDValue Root;
4022  bool ConstantMemory = false;
4023  if (isVolatile || NumValues > MaxParallelChains)
4024  // Serialize volatile loads with other side effects.
4025  Root = getRoot();
4026  else if (AA &&
4027  AA->pointsToConstantMemory(MemoryLocation(
4028  SV,
4030  AAInfo))) {
4031  // Do not serialize (non-volatile) loads of constant memory with anything.
4032  Root = DAG.getEntryNode();
4033  ConstantMemory = true;
4034  } else {
4035  // Do not serialize non-volatile loads against each other.
4036  Root = DAG.getRoot();
4037  }
4038 
4039  SDLoc dl = getCurSDLoc();
4040 
4041  if (isVolatile)
4042  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4043 
4044  // An aggregate load cannot wrap around the address space, so offsets to its
4045  // parts don't wrap either.
4046  SDNodeFlags Flags;
4047  Flags.setNoUnsignedWrap(true);
4048 
4049  SmallVector<SDValue, 4> Values(NumValues);
4050  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4051  EVT PtrVT = Ptr.getValueType();
4052  unsigned ChainI = 0;
4053  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4054  // Serializing loads here may result in excessive register pressure, and
4055  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4056  // could recover a bit by hoisting nodes upward in the chain by recognizing
4057  // they are side-effect free or do not alias. The optimizer should really
4058  // avoid this case by converting large object/array copies to llvm.memcpy
4059  // (MaxParallelChains should always remain as failsafe).
4060  if (ChainI == MaxParallelChains) {
4061  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4062  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4063  makeArrayRef(Chains.data(), ChainI));
4064  Root = Chain;
4065  ChainI = 0;
4066  }
4067  SDValue A = DAG.getNode(ISD::ADD, dl,
4068  PtrVT, Ptr,
4069  DAG.getConstant(Offsets[i], dl, PtrVT),
4070  Flags);
4071  auto MMOFlags = MachineMemOperand::MONone;
4072  if (isVolatile)
4073  MMOFlags |= MachineMemOperand::MOVolatile;
4074  if (isNonTemporal)
4076  if (isInvariant)
4077  MMOFlags |= MachineMemOperand::MOInvariant;
4078  if (isDereferenceable)
4080  MMOFlags |= TLI.getMMOFlags(I);
4081 
4082  SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4083  MachinePointerInfo(SV, Offsets[i]), Alignment,
4084  MMOFlags, AAInfo, Ranges);
4085  Chains[ChainI] = L.getValue(1);
4086 
4087  if (MemVTs[i] != ValueVTs[i])
4088  L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4089 
4090  Values[i] = L;
4091  }
4092 
4093  if (!ConstantMemory) {
4094  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4095  makeArrayRef(Chains.data(), ChainI));
4096  if (isVolatile)
4097  DAG.setRoot(Chain);
4098  else
4099  PendingLoads.push_back(Chain);
4100  }
4101 
4102  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4103  DAG.getVTList(ValueVTs), Values));
4104 }
4105 
4106 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4108  "call visitStoreToSwiftError when backend supports swifterror");
4109 
4112  const Value *SrcV = I.getOperand(0);
4114  SrcV->getType(), ValueVTs, &Offsets);
4115  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4116  "expect a single EVT for swifterror");
4117 
4118  SDValue Src = getValue(SrcV);
4119  // Create a virtual register, then update the virtual register.
4120  Register VReg =
4121  SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4122  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4123  // Chain can be getRoot or getControlRoot.
4124  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4125  SDValue(Src.getNode(), Src.getResNo()));
4126  DAG.setRoot(CopyNode);
4127 }
4128 
4129 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4131  "call visitLoadFromSwiftError when backend supports swifterror");
4132 
4133  assert(!I.isVolatile() &&
4134  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
4135  I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
4136  "Support volatile, non temporal, invariant for load_from_swift_error");
4137 
4138  const Value *SV = I.getOperand(0);
4139  Type *Ty = I.getType();
4140  AAMDNodes AAInfo;
4141  I.getAAMetadata(AAInfo);
4142  assert(
4143  (!AA ||
4144  !AA->pointsToConstantMemory(MemoryLocation(
4146  AAInfo))) &&
4147  "load_from_swift_error should not be constant memory");
4148 
4152  ValueVTs, &Offsets);
4153  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4154  "expect a single EVT for swifterror");
4155 
4156  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4157  SDValue L = DAG.getCopyFromReg(
4158  getRoot(), getCurSDLoc(),
4159  SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4160 
4161  setValue(&I, L);
4162 }
4163 
4164 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4165  if (I.isAtomic())
4166  return visitAtomicStore(I);
4167 
4168  const Value *SrcV = I.getOperand(0);
4169  const Value *PtrV = I.getOperand(1);
4170 
4171  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4172  if (TLI.supportSwiftError()) {
4173  // Swifterror values can come from either a function parameter with
4174  // swifterror attribute or an alloca with swifterror attribute.
4175  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4176  if (Arg->hasSwiftErrorAttr())
4177  return visitStoreToSwiftError(I);
4178  }
4179 
4180  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4181  if (Alloca->isSwiftError())
4182  return visitStoreToSwiftError(I);
4183  }
4184  }
4185 
4186  SmallVector<EVT, 4> ValueVTs, MemVTs;
4189  SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4190  unsigned NumValues = ValueVTs.size();
4191  if (NumValues == 0)
4192  return;
4193 
4194  // Get the lowered operands. Note that we do this after
4195  // checking if NumResults is zero, because with zero results
4196  // the operands won't have values in the map.
4197  SDValue Src = getValue(SrcV);
4198  SDValue Ptr = getValue(PtrV);
4199 
4200  SDValue Root = getRoot();
4201  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4202  SDLoc dl = getCurSDLoc();
4203  EVT PtrVT = Ptr.getValueType();
4204  unsigned Alignment = I.getAlignment();
4205  AAMDNodes AAInfo;
4206  I.getAAMetadata(AAInfo);
4207 
4208  auto MMOFlags = MachineMemOperand::MONone;
4209  if (I.isVolatile())
4210  MMOFlags |= MachineMemOperand::MOVolatile;
4211  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
4213  MMOFlags |= TLI.getMMOFlags(I);
4214 
4215  // An aggregate load cannot wrap around the address space, so offsets to its
4216  // parts don't wrap either.
4217  SDNodeFlags Flags;
4218  Flags.setNoUnsignedWrap(true);
4219 
4220  unsigned ChainI = 0;
4221  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4222  // See visitLoad comments.
4223  if (ChainI == MaxParallelChains) {
4224  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4225  makeArrayRef(Chains.data(), ChainI));
4226  Root = Chain;
4227  ChainI = 0;
4228  }
4229  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
4230  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
4231  SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4232  if (MemVTs[i] != ValueVTs[i])
4233  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4234  SDValue St =
4235  DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4236  Alignment, MMOFlags, AAInfo);
4237  Chains[ChainI] = St;
4238  }
4239 
4240  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4241  makeArrayRef(Chains.data(), ChainI));
4242  DAG.setRoot(StoreNode);
4243 }
4244 
4245 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4246  bool IsCompressing) {
4247  SDLoc sdl = getCurSDLoc();
4248 
4249  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4250  unsigned& Alignment) {
4251  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4252  Src0 = I.getArgOperand(0);
4253  Ptr = I.getArgOperand(1);
4254  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
4255  Mask = I.getArgOperand(3);
4256  };
4257  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4258  unsigned& Alignment) {
4259  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4260  Src0 = I.getArgOperand(0);
4261  Ptr = I.getArgOperand(1);
4262  Mask = I.getArgOperand(2);
4263  Alignment = 0;
4264  };
4265 
4266  Value *PtrOperand, *MaskOperand, *Src0Operand;
4267  unsigned Alignment;
4268  if (IsCompressing)
4269  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4270  else
4271  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4272 
4273  SDValue Ptr = getValue(PtrOperand);
4274  SDValue Src0 = getValue(Src0Operand);
4275  SDValue Mask = getValue(MaskOperand);
4276 
4277  EVT VT = Src0.getValueType();
4278  if (!Alignment)
4279  Alignment = DAG.getEVTAlignment(VT);
4280 
4281  AAMDNodes AAInfo;
4282  I.getAAMetadata(AAInfo);
4283 
4284  MachineMemOperand *MMO =
4285  DAG.getMachineFunction().
4288  Alignment, AAInfo);
4289  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
4290  MMO, false /* Truncating */,
4291  IsCompressing);
4292  DAG.setRoot(StoreNode);
4293  setValue(&I, StoreNode);
4294 }
4295 
4296 // Get a uniform base for the Gather/Scatter intrinsic.
4297 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4298 // We try to represent it as a base pointer + vector of indices.
4299 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4300 // The first operand of the GEP may be a single pointer or a vector of pointers
4301 // Example:
4302 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4303 // or
4304 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4305 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4306 //
4307 // When the first GEP operand is a single pointer - it is the uniform base we
4308 // are looking for. If first operand of the GEP is a splat vector - we
4309 // extract the splat value and use it as a uniform base.
4310 // In all other cases the function returns 'false'.
4311 static bool getUniformBase(const Value *&Ptr, SDValue &Base, SDValue &Index,
4312  ISD::MemIndexType &IndexType, SDValue &Scale,
4313  SelectionDAGBuilder *SDB) {
4314  SelectionDAG& DAG = SDB->DAG;
4315  LLVMContext &Context = *DAG.getContext();
4316 
4317  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
4319  if (!GEP)
4320  return false;
4321 
4322  const Value *GEPPtr = GEP->getPointerOperand();
4323  if (!GEPPtr->getType()->isVectorTy())
4324  Ptr = GEPPtr;
4325  else if (!(Ptr = getSplatValue(GEPPtr)))
4326  return false;
4327 
4328  unsigned FinalIndex = GEP->getNumOperands() - 1;
4329  Value *IndexVal = GEP->getOperand(FinalIndex);
4330 
4331  // Ensure all the other indices are 0.
4332  for (unsigned i = 1; i < FinalIndex; ++i) {
4333  auto *C = dyn_cast<Constant>(GEP->getOperand(i));
4334  if (!C)
4335  return false;
4336  if (isa<VectorType>(C->getType()))
4337  C = C->getSplatValue();
4338  auto *CI = dyn_cast_or_null<ConstantInt>(C);
4339  if (!CI || !CI->isZero())
4340  return false;
4341  }
4342 
4343  // The operands of the GEP may be defined in another basic block.
4344  // In this case we'll not find nodes for the operands.
4345  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
4346  return false;
4347 
4348  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4349  const DataLayout &DL = DAG.getDataLayout();
4350  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4351  SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4352  Base = SDB->getValue(Ptr);
4353  Index = SDB->getValue(IndexVal);
4354  IndexType = ISD::SIGNED_SCALED;
4355 
4356  if (!Index.getValueType().isVector()) {
4357  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4358  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4359  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4360  }
4361  return true;
4362 }
4363 
4364 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4365  SDLoc sdl = getCurSDLoc();
4366 
4367  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4368  const Value *Ptr = I.getArgOperand(1);
4369  SDValue Src0 = getValue(I.getArgOperand(0));
4370  SDValue Mask = getValue(I.getArgOperand(3));
4371  EVT VT = Src0.getValueType();
4372  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4373  if (!Alignment)
4374  Alignment = DAG.getEVTAlignment(VT);
4375  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4376 
4377  AAMDNodes AAInfo;
4378  I.getAAMetadata(AAInfo);
4379 
4380  SDValue Base;
4381  SDValue Index;
4382  ISD::MemIndexType IndexType;
4383  SDValue Scale;
4384  const Value *BasePtr = Ptr;
4385  bool UniformBase = getUniformBase(BasePtr, Base, Index, IndexType, Scale,
4386  this);
4387 
4388  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4391  MachineMemOperand::MOStore, VT.getStoreSize(),
4392  Alignment, AAInfo);
4393  if (!UniformBase) {
4394  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4395  Index = getValue(Ptr);
4396  IndexType = ISD::SIGNED_SCALED;
4397  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4398  }
4399  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4400  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4401  Ops, MMO, IndexType);
4402  DAG.setRoot(Scatter);
4403  setValue(&I, Scatter);
4404 }
4405 
4406 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4407  SDLoc sdl = getCurSDLoc();
4408 
4409  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4410  unsigned& Alignment) {
4411  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4412  Ptr = I.getArgOperand(0);
4413  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4414  Mask = I.getArgOperand(2);
4415  Src0 = I.getArgOperand(3);
4416  };
4417  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4418  unsigned& Alignment) {
4419  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4420  Ptr = I.getArgOperand(0);
4421  Alignment = 0;
4422  Mask = I.getArgOperand(1);
4423  Src0 = I.getArgOperand(2);
4424  };
4425 
4426  Value *PtrOperand, *MaskOperand, *Src0Operand;
4427  unsigned Alignment;
4428  if (IsExpanding)
4429  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4430  else
4431  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4432 
4433  SDValue Ptr = getValue(PtrOperand);
4434  SDValue Src0 = getValue(Src0Operand);
4435  SDValue Mask = getValue(MaskOperand);
4436 
4437  EVT VT = Src0.getValueType();
4438  if (!Alignment)
4439  Alignment = DAG.getEVTAlignment(VT);
4440 
4441  AAMDNodes AAInfo;
4442  I.getAAMetadata(AAInfo);
4443  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4444 
4445  // Do not serialize masked loads of constant memory with anything.
4446  bool AddToChain =
4447  !AA || !AA->pointsToConstantMemory(MemoryLocation(
4448  PtrOperand,
4451  AAInfo));
4452  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4453 
4454  MachineMemOperand *MMO =
4455  DAG.getMachineFunction().
4458  Alignment, AAInfo, Ranges);
4459 
4460  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4461  ISD::NON_EXTLOAD, IsExpanding);
4462  if (AddToChain)
4463  PendingLoads.push_back(Load.getValue(1));
4464  setValue(&I, Load);
4465 }
4466 
4467 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4468  SDLoc sdl = getCurSDLoc();
4469 
4470  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4471  const Value *Ptr = I.getArgOperand(0);
4472  SDValue Src0 = getValue(I.getArgOperand(3));
4473  SDValue Mask = getValue(I.getArgOperand(2));
4474 
4475  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4476  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4477  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4478  if (!Alignment)
4479  Alignment = DAG.getEVTAlignment(VT);
4480 
4481  AAMDNodes AAInfo;
4482  I.getAAMetadata(AAInfo);
4483  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4484 
4485  SDValue Root = DAG.getRoot();
4486  SDValue Base;
4487  SDValue Index;
4488  ISD::MemIndexType IndexType;
4489  SDValue Scale;
4490  const Value *BasePtr = Ptr;
4491  bool UniformBase = getUniformBase(BasePtr, Base, Index, IndexType, Scale,
4492  this);
4493  bool ConstantMemory = false;
4494  if (UniformBase && AA &&
4495  AA->pointsToConstantMemory(
4496  MemoryLocation(BasePtr,
4499  AAInfo))) {
4500  // Do not serialize (non-volatile) loads of constant memory with anything.
4501  Root = DAG.getEntryNode();
4502  ConstantMemory = true;
4503  }
4504 
4505  MachineMemOperand *MMO =
4506  DAG.getMachineFunction().
4507  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4509  Alignment, AAInfo, Ranges);
4510 
4511  if (!UniformBase) {
4512  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4513  Index = getValue(Ptr);
4514  IndexType = ISD::SIGNED_SCALED;
4515  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4516  }
4517  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4518  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4519  Ops, MMO, IndexType);
4520 
4521  SDValue OutChain = Gather.getValue(1);
4522  if (!ConstantMemory)
4523  PendingLoads.push_back(OutChain);
4524  setValue(&I, Gather);
4525 }
4526 
4527 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4528  SDLoc dl = getCurSDLoc();
4529  AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4530  AtomicOrdering FailureOrdering = I.getFailureOrdering();
4531  SyncScope::ID SSID = I.getSyncScopeID();
4532 
4533  SDValue InChain = getRoot();
4534 
4535  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4536  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4537 
4538  auto Alignment = DAG.getEVTAlignment(MemVT);
4539 
4541  if (I.isVolatile())
4543  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4544 
4545  MachineFunction &MF = DAG.getMachineFunction();
4546  MachineMemOperand *MMO =
4548  Flags, MemVT.getStoreSize(), Alignment,
4549  AAMDNodes(), nullptr, SSID, SuccessOrdering,
4550  FailureOrdering);
4551 
4553  dl, MemVT, VTs, InChain,
4554  getValue(I.getPointerOperand()),
4555  getValue(I.getCompareOperand()),
4556  getValue(I.getNewValOperand()), MMO);
4557 
4558  SDValue OutChain = L.getValue(2);
4559 
4560  setValue(&I, L);
4561  DAG.setRoot(OutChain);
4562 }
4563 
4564 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4565  SDLoc dl = getCurSDLoc();
4566  ISD::NodeType NT;
4567  switch (I.getOperation()) {
4568  default: llvm_unreachable("Unknown atomicrmw operation");
4569  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4570  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4571  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4572  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4573  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4574  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4575  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4576  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4577  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4578  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4579  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4580  case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4581  case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4582  }
4583  AtomicOrdering Ordering = I.getOrdering();
4584  SyncScope::ID SSID = I.getSyncScopeID();
4585 
4586  SDValue InChain = getRoot();
4587 
4588  auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4589  auto Alignment = DAG.getEVTAlignment(MemVT);
4590 
4592  if (I.isVolatile())
4594  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4595 
4596  MachineFunction &MF = DAG.getMachineFunction();
4597  MachineMemOperand *MMO =
4599  MemVT.getStoreSize(), Alignment, AAMDNodes(),
4600  nullptr, SSID, Ordering);
4601 
4602  SDValue L =
4603  DAG.getAtomic(NT, dl, MemVT, InChain,
4604  getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4605  MMO);
4606 
4607  SDValue OutChain = L.getValue(1);
4608 
4609  setValue(&I, L);
4610  DAG.setRoot(OutChain);
4611 }
4612 
4613 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4614  SDLoc dl = getCurSDLoc();
4615  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4616  SDValue Ops[3];
4617  Ops[0] = getRoot();
4618  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4619  TLI.getFenceOperandTy(DAG.getDataLayout()));
4620  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4621  TLI.getFenceOperandTy(DAG.getDataLayout()));
4622  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4623 }
4624 
4625 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4626  SDLoc dl = getCurSDLoc();
4627  AtomicOrdering Order = I.getOrdering();
4628  SyncScope::ID SSID = I.getSyncScopeID();
4629 
4630  SDValue InChain = getRoot();
4631 
4632  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4633  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4634  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4635 
4636  if (!TLI.supportsUnalignedAtomics() &&
4637  I.getAlignment() < MemVT.getSizeInBits() / 8)
4638  report_fatal_error("Cannot generate unaligned atomic load");
4639 
4640  auto Flags = MachineMemOperand::MOLoad;
4641  if (I.isVolatile())
4643  if (I.getMetadata(LLVMContext::MD_invariant_load) != nullptr)
4646  DAG.getDataLayout()))
4648 
4649  Flags |= TLI.getMMOFlags(I);
4650 
4651  MachineMemOperand *MMO =
4652  DAG.getMachineFunction().
4654  Flags, MemVT.getStoreSize(),
4655  I.getAlignment() ? I.getAlignment() :
4656  DAG.getEVTAlignment(MemVT),
4657  AAMDNodes(), nullptr, SSID, Order);
4658 
4659  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4660  SDValue L =
4661  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4662  getValue(I.getPointerOperand()), MMO);
4663 
4664  SDValue OutChain = L.getValue(1);
4665  if (MemVT != VT)
4666  L = DAG.getPtrExtOrTrunc(L, dl, VT);
4667 
4668  setValue(&I, L);
4669  DAG.setRoot(OutChain);
4670 }
4671 
4672 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4673  SDLoc dl = getCurSDLoc();
4674 
4675  AtomicOrdering Ordering = I.getOrdering();
4676  SyncScope::ID SSID = I.getSyncScopeID();
4677 
4678  SDValue InChain = getRoot();
4679 
4680  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4681  EVT MemVT =
4683 
4684  if (I.getAlignment() < MemVT.getSizeInBits() / 8)
4685  report_fatal_error("Cannot generate unaligned atomic store");
4686 
4687  auto Flags = MachineMemOperand::MOStore;
4688  if (I.isVolatile())
4690  Flags |= TLI.getMMOFlags(I);
4691 
4692  MachineFunction &MF = DAG.getMachineFunction();
4693  MachineMemOperand *MMO =
4695  MemVT.getStoreSize(), I.getAlignment(), AAMDNodes(),
4696  nullptr, SSID, Ordering);
4697 
4698  SDValue Val = getValue(I.getValueOperand());
4699  if (Val.getValueType() != MemVT)
4700  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4701 
4702  SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4703  getValue(I.getPointerOperand()), Val, MMO);
4704 
4705 
4706  DAG.setRoot(OutChain);
4707 }
4708 
4709 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4710 /// node.
4711 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4712  unsigned Intrinsic) {
4713  // Ignore the callsite's attributes. A specific call site may be marked with
4714  // readnone, but the lowering code will expect the chain based on the
4715  // definition.
4716  const Function *F = I.getCalledFunction();
4717  bool HasChain = !F->doesNotAccessMemory();
4718  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4719 
4720  // Build the operand list.
4722  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4723  if (OnlyLoad) {
4724  // We don't need to serialize loads against other loads.
4725  Ops.push_back(DAG.getRoot());
4726  } else {
4727  Ops.push_back(getRoot());
4728  }
4729  }
4730 
4731  // Info is set by getTgtMemInstrinsic
4732  TargetLowering::IntrinsicInfo Info;
4733  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4734  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4735  DAG.getMachineFunction(),
4736  Intrinsic);
4737 
4738  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4739  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4740  Info.opc == ISD::INTRINSIC_W_CHAIN)
4741  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4742  TLI.getPointerTy(DAG.getDataLayout())));
4743 
4744  // Add all operands of the call to the operand list.
4745  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4746  SDValue Op = getValue(I.getArgOperand(i));
4747  Ops.push_back(Op);
4748  }
4749 
4751  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4752 
4753  if (HasChain)
4754  ValueVTs.push_back(MVT::Other);
4755 
4756  SDVTList VTs = DAG.getVTList(ValueVTs);
4757 
4758  // Create the node.
4759  SDValue Result;
4760  if (IsTgtIntrinsic) {
4761  // This is target intrinsic that touches memory
4762  AAMDNodes AAInfo;
4763  I.getAAMetadata(AAInfo);
4764  Result = DAG.getMemIntrinsicNode(
4765  Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4766  MachinePointerInfo(Info.ptrVal, Info.offset),
4767  Info.align ? Info.align->value() : 0, Info.flags, Info.size, AAInfo);
4768  } else if (!HasChain) {
4769  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4770  } else if (!I.getType()->isVoidTy()) {
4771  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4772  } else {
4773  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4774  }
4775 
4776  if (HasChain) {
4777  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4778  if (OnlyLoad)
4779  PendingLoads.push_back(Chain);
4780  else
4781  DAG.setRoot(Chain);
4782  }
4783 
4784  if (!I.getType()->isVoidTy()) {
4785  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4786  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4787  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4788  } else
4789  Result = lowerRangeToAssertZExt(DAG, I, Result);
4790 
4791  setValue(&I, Result);
4792  }
4793 }
4794 
4795 /// GetSignificand - Get the significand and build it into a floating-point
4796 /// number with exponent of 1:
4797 ///
4798 /// Op = (Op & 0x007fffff) | 0x3f800000;
4799 ///
4800 /// where Op is the hexadecimal representation of floating point value.
4802  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4803  DAG.getConstant(0x007fffff, dl, MVT::i32));
4804  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4805  DAG.getConstant(0x3f800000, dl, MVT::i32));
4806  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4807 }
4808 
4809 /// GetExponent - Get the exponent:
4810 ///
4811 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4812 ///
4813 /// where Op is the hexadecimal representation of floating point value.
4815  const TargetLowering &TLI, const SDLoc &dl) {
4816  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4817  DAG.getConstant(0x7f800000, dl, MVT::i32));
4818  SDValue t1 = DAG.getNode(
4819  ISD::SRL, dl, MVT::i32, t0,
4820  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4821  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4822  DAG.getConstant(127, dl, MVT::i32));
4823  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4824 }
4825 
4826 /// getF32Constant - Get 32-bit floating point constant.
4827 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4828  const SDLoc &dl) {
4829  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4830  MVT::f32);
4831 }
4832 
4834  SelectionDAG &DAG) {
4835  // TODO: What fast-math-flags should be set on the floating-point nodes?
4836 
4837  // IntegerPartOfX = ((int32_t)(t0);
4838  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4839 
4840  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4841  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4842  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4843 
4844  // IntegerPartOfX <<= 23;
4845  IntegerPartOfX = DAG.getNode(
4846  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4848  DAG.getDataLayout())));
4849 
4850  SDValue TwoToFractionalPartOfX;
4851  if (LimitFloatPrecision <= 6) {
4852  // For floating-point precision of 6:
4853  //
4854  // TwoToFractionalPartOfX =
4855  // 0.997535578f +
4856  // (0.735607626f + 0.252464424f * x) * x;
4857  //
4858  // error 0.0144103317, which is 6 bits
4859  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4860  getF32Constant(DAG, 0x3e814304, dl));
4861  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4862  getF32Constant(DAG, 0x3f3c50c8, dl));
4863  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4864  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4865  getF32Constant(DAG, 0x3f7f5e7e, dl));
4866  } else if (LimitFloatPrecision <= 12) {
4867  // For floating-point precision of 12:
4868  //
4869  // TwoToFractionalPartOfX =
4870  // 0.999892986f +
4871  // (0.696457318f +
4872  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4873  //
4874  // error 0.000107046256, which is 13 to 14 bits
4875  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4876  getF32Constant(DAG, 0x3da235e3, dl));
4877  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4878  getF32Constant(DAG, 0x3e65b8f3, dl));
4879  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4880  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4881  getF32Constant(DAG, 0x3f324b07, dl));
4882  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4883  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4884  getF32Constant(DAG, 0x3f7ff8fd, dl));
4885  } else { // LimitFloatPrecision <= 18
4886  // For floating-point precision of 18:
4887  //
4888  // TwoToFractionalPartOfX =
4889  // 0.999999982f +
4890  // (0.693148872f +
4891  // (0.240227044f +
4892  // (0.554906021e-1f +
4893  // (0.961591928e-2f +
4894  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4895  // error 2.47208000*10^(-7), which is better than 18 bits
4896  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4897  getF32Constant(DAG, 0x3924b03e, dl));
4898  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4899  getF32Constant(DAG, 0x3ab24b87, dl));
4900  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4901  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4902  getF32Constant(DAG, 0x3c1d8c17, dl));
4903  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4904  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4905  getF32Constant(DAG, 0x3d634a1d, dl));
4906  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4907  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4908  getF32Constant(DAG, 0x3e75fe14, dl));
4909  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4910  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4911  getF32Constant(DAG, 0x3f317234, dl));
4912  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4913  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4914  getF32Constant(DAG, 0x3f800000, dl));
4915  }
4916 
4917  // Add the exponent into the result in integer domain.
4918  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4919  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4920  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4921 }
4922 
4923 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4924 /// limited-precision mode.
4925 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4926  const TargetLowering &TLI) {
4927  if (Op.getValueType() == MVT::f32 &&
4928  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4929 
4930  // Put the exponent in the right bit position for later addition to the
4931  // final result:
4932  //
4933  // #define LOG2OFe 1.4426950f
4934  // t0 = Op * LOG2OFe
4935 
4936  // TODO: What fast-math-flags should be set here?
4937  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4938  getF32Constant(DAG, 0x3fb8aa3b, dl));
4939  return getLimitedPrecisionExp2(t0, dl, DAG);
4940  }
4941 
4942  // No special expansion.
4943  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4944 }
4945 
4946 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4947 /// limited-precision mode.
4948 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4949  const TargetLowering &TLI) {
4950  // TODO: What fast-math-flags should be set on the floating-point nodes?
4951 
4952  if (Op.getValueType() == MVT::f32 &&
4953  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4954  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4955 
4956  // Scale the exponent by log(2) [0.69314718f].
4957  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4958  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4959  getF32Constant(DAG, 0x3f317218, dl));
4960 
4961  // Get the significand and build it into a floating-point number with
4962  // exponent of 1.
4963  SDValue X = GetSignificand(DAG, Op1, dl);
4964 
4965  SDValue LogOfMantissa;
4966  if (LimitFloatPrecision <= 6) {
4967  // For floating-point precision of 6:
4968  //
4969  // LogofMantissa =
4970  // -1.1609546f +
4971  // (1.4034025f - 0.23903021f * x) * x;
4972  //
4973  // error 0.0034276066, which is better than 8 bits
4974  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4975  getF32Constant(DAG, 0xbe74c456, dl));
4976  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4977  getF32Constant(DAG, 0x3fb3a2b1, dl));
4978  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4979  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4980  getF32Constant(DAG, 0x3f949a29, dl));
4981  } else if (LimitFloatPrecision <= 12) {
4982  // For floating-point precision of 12:
4983  //
4984  // LogOfMantissa =
4985  // -1.7417939f +
4986  // (2.8212026f +
4987  // (-1.4699568f +
4988  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4989  //
4990  // error 0.000061011436, which is 14 bits
4991  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4992  getF32Constant(DAG, 0xbd67b6d6, dl));
4993  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4994  getF32Constant(DAG, 0x3ee4f4b8, dl));
4995  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4996  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4997  getF32Constant(DAG, 0x3fbc278b, dl));
4998  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4999  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5000  getF32Constant(DAG, 0x40348e95, dl));
5001  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5002  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5003  getF32Constant(DAG, 0x3fdef31a, dl));
5004  } else { // LimitFloatPrecision <= 18
5005  // For floating-point precision of 18:
5006  //
5007  // LogOfMantissa =
5008  // -2.1072184f +
5009  // (4.2372794f +
5010  // (-3.7029485f +
5011  // (2.2781945f +
5012  // (-0.87823314f +
5013  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5014  //
5015  // error 0.0000023660568, which is better than 18 bits
5016  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5017  getF32Constant(DAG, 0xbc91e5ac, dl));
5018  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5019  getF32Constant(DAG, 0x3e4350aa, dl));
5020  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5021  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5022  getF32Constant(DAG, 0x3f60d3e3, dl));
5023  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5024  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5025  getF32Constant(DAG, 0x4011cdf0, dl));
5026  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5027  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5028  getF32Constant(DAG, 0x406cfd1c, dl));
5029  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5030  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5031  getF32Constant(DAG, 0x408797cb, dl));
5032  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5033  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5034  getF32Constant(DAG, 0x4006dcab, dl));
5035  }
5036 
5037  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5038  }
5039 
5040  // No special expansion.
5041  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
5042 }
5043 
5044 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5045 /// limited-precision mode.
5046 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5047  const TargetLowering &TLI) {
5048  // TODO: What fast-math-flags should be set on the floating-point nodes?
5049 
5050  if (Op.getValueType() == MVT::f32 &&
5051  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5052  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5053 
5054  // Get the exponent.
5055  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5056 
5057  // Get the significand and build it into a floating-point number with
5058  // exponent of 1.
5059  SDValue X = GetSignificand(DAG, Op1, dl);
5060 
5061  // Different possible minimax approximations of significand in
5062  // floating-point for various degrees of accuracy over [1,2].
5063  SDValue Log2ofMantissa;
5064  if (LimitFloatPrecision <= 6) {
5065