LLVM  14.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/Triple.h"
25 #include "llvm/ADT/Twine.h"
31 #include "llvm/Analysis/Loads.h"
37 #include "llvm/CodeGen/Analysis.h"
53 #include "llvm/CodeGen/StackMaps.h"
61 #include "llvm/IR/Argument.h"
62 #include "llvm/IR/Attributes.h"
63 #include "llvm/IR/BasicBlock.h"
64 #include "llvm/IR/CFG.h"
65 #include "llvm/IR/CallingConv.h"
66 #include "llvm/IR/Constant.h"
67 #include "llvm/IR/ConstantRange.h"
68 #include "llvm/IR/Constants.h"
69 #include "llvm/IR/DataLayout.h"
71 #include "llvm/IR/DerivedTypes.h"
72 #include "llvm/IR/DiagnosticInfo.h"
73 #include "llvm/IR/Function.h"
75 #include "llvm/IR/InlineAsm.h"
76 #include "llvm/IR/InstrTypes.h"
77 #include "llvm/IR/Instructions.h"
78 #include "llvm/IR/IntrinsicInst.h"
79 #include "llvm/IR/Intrinsics.h"
80 #include "llvm/IR/IntrinsicsAArch64.h"
81 #include "llvm/IR/IntrinsicsWebAssembly.h"
82 #include "llvm/IR/LLVMContext.h"
83 #include "llvm/IR/Metadata.h"
84 #include "llvm/IR/Module.h"
85 #include "llvm/IR/Operator.h"
86 #include "llvm/IR/PatternMatch.h"
87 #include "llvm/IR/Statepoint.h"
88 #include "llvm/IR/Type.h"
89 #include "llvm/IR/User.h"
90 #include "llvm/IR/Value.h"
91 #include "llvm/MC/MCContext.h"
92 #include "llvm/MC/MCSymbol.h"
94 #include "llvm/Support/Casting.h"
96 #include "llvm/Support/Compiler.h"
97 #include "llvm/Support/Debug.h"
104 #include <cstddef>
105 #include <cstring>
106 #include <iterator>
107 #include <limits>
108 #include <numeric>
109 #include <tuple>
110 
111 using namespace llvm;
112 using namespace PatternMatch;
113 using namespace SwitchCG;
114 
115 #define DEBUG_TYPE "isel"
116 
117 /// LimitFloatPrecision - Generate low-precision inline sequences for
118 /// some float libcalls (6, 8 or 12 bits).
119 static unsigned LimitFloatPrecision;
120 
121 static cl::opt<bool>
122  InsertAssertAlign("insert-assert-align", cl::init(true),
123  cl::desc("Insert the experimental `assertalign` node."),
125 
127  LimitFPPrecision("limit-float-precision",
128  cl::desc("Generate low-precision inline sequences "
129  "for some float libcalls"),
131  cl::init(0));
132 
134  "switch-peel-threshold", cl::Hidden, cl::init(66),
135  cl::desc("Set the case probability threshold for peeling the case from a "
136  "switch statement. A value greater than 100 will void this "
137  "optimization"));
138 
139 // Limit the width of DAG chains. This is important in general to prevent
140 // DAG-based analysis from blowing up. For example, alias analysis and
141 // load clustering may not complete in reasonable time. It is difficult to
142 // recognize and avoid this situation within each individual analysis, and
143 // future analyses are likely to have the same behavior. Limiting DAG width is
144 // the safe approach and will be especially important with global DAGs.
145 //
146 // MaxParallelChains default is arbitrarily high to avoid affecting
147 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148 // sequence over this should have been converted to llvm.memcpy by the
149 // frontend. It is easy to induce this behavior with .ll code such as:
150 // %buffer = alloca [4096 x i8]
151 // %data = load [4096 x i8]* %argPtr
152 // store [4096 x i8] %data, [4096 x i8]* %buffer
153 static const unsigned MaxParallelChains = 64;
154 
156  const SDValue *Parts, unsigned NumParts,
157  MVT PartVT, EVT ValueVT, const Value *V,
159 
160 /// getCopyFromParts - Create a value that contains the specified legal parts
161 /// combined into the value they represent. If the parts combine to a type
162 /// larger than ValueVT then AssertOp can be used to specify whether the extra
163 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
164 /// (ISD::AssertSext).
166  const SDValue *Parts, unsigned NumParts,
167  MVT PartVT, EVT ValueVT, const Value *V,
168  Optional<CallingConv::ID> CC = None,
169  Optional<ISD::NodeType> AssertOp = None) {
170  // Let the target assemble the parts if it wants to
171  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
172  if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
173  PartVT, ValueVT, CC))
174  return Val;
175 
176  if (ValueVT.isVector())
177  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
178  CC);
179 
180  assert(NumParts > 0 && "No parts to assemble!");
181  SDValue Val = Parts[0];
182 
183  if (NumParts > 1) {
184  // Assemble the value from multiple parts.
185  if (ValueVT.isInteger()) {
186  unsigned PartBits = PartVT.getSizeInBits();
187  unsigned ValueBits = ValueVT.getSizeInBits();
188 
189  // Assemble the power of 2 part.
190  unsigned RoundParts =
191  (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
192  unsigned RoundBits = PartBits * RoundParts;
193  EVT RoundVT = RoundBits == ValueBits ?
194  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
195  SDValue Lo, Hi;
196 
197  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
198 
199  if (RoundParts > 2) {
200  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
201  PartVT, HalfVT, V);
202  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
203  RoundParts / 2, PartVT, HalfVT, V);
204  } else {
205  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
206  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
207  }
208 
209  if (DAG.getDataLayout().isBigEndian())
210  std::swap(Lo, Hi);
211 
212  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
213 
214  if (RoundParts < NumParts) {
215  // Assemble the trailing non-power-of-2 part.
216  unsigned OddParts = NumParts - RoundParts;
217  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
218  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
219  OddVT, V, CC);
220 
221  // Combine the round and odd parts.
222  Lo = Val;
223  if (DAG.getDataLayout().isBigEndian())
224  std::swap(Lo, Hi);
225  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
226  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
227  Hi =
228  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
229  DAG.getConstant(Lo.getValueSizeInBits(), DL,
230  TLI.getPointerTy(DAG.getDataLayout())));
231  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
232  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
233  }
234  } else if (PartVT.isFloatingPoint()) {
235  // FP split into multiple FP parts (for ppcf128)
236  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
237  "Unexpected split");
238  SDValue Lo, Hi;
239  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
240  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
241  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
242  std::swap(Lo, Hi);
243  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
244  } else {
245  // FP split into integer parts (soft fp)
246  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
247  !PartVT.isVector() && "Unexpected split");
248  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
249  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
250  }
251  }
252 
253  // There is now one part, held in Val. Correct it to match ValueVT.
254  // PartEVT is the type of the register class that holds the value.
255  // ValueVT is the type of the inline asm operation.
256  EVT PartEVT = Val.getValueType();
257 
258  if (PartEVT == ValueVT)
259  return Val;
260 
261  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
262  ValueVT.bitsLT(PartEVT)) {
263  // For an FP value in an integer part, we need to truncate to the right
264  // width first.
265  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
266  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
267  }
268 
269  // Handle types that have the same size.
270  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
271  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
272 
273  // Handle types with different sizes.
274  if (PartEVT.isInteger() && ValueVT.isInteger()) {
275  if (ValueVT.bitsLT(PartEVT)) {
276  // For a truncate, see if we have any information to
277  // indicate whether the truncated bits will always be
278  // zero or sign-extension.
279  if (AssertOp.hasValue())
280  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
281  DAG.getValueType(ValueVT));
282  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
283  }
284  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
285  }
286 
287  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
288  // FP_ROUND's are always exact here.
289  if (ValueVT.bitsLT(Val.getValueType()))
290  return DAG.getNode(
291  ISD::FP_ROUND, DL, ValueVT, Val,
292  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
293 
294  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
295  }
296 
297  // Handle MMX to a narrower integer type by bitcasting MMX to integer and
298  // then truncating.
299  if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
300  ValueVT.bitsLT(PartEVT)) {
301  Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
302  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
303  }
304 
305  report_fatal_error("Unknown mismatch in getCopyFromParts!");
306 }
307 
309  const Twine &ErrMsg) {
310  const Instruction *I = dyn_cast_or_null<Instruction>(V);
311  if (!V)
312  return Ctx.emitError(ErrMsg);
313 
314  const char *AsmError = ", possible invalid constraint for vector type";
315  if (const CallInst *CI = dyn_cast<CallInst>(I))
316  if (CI->isInlineAsm())
317  return Ctx.emitError(I, ErrMsg + AsmError);
318 
319  return Ctx.emitError(I, ErrMsg);
320 }
321 
322 /// getCopyFromPartsVector - Create a value that contains the specified legal
323 /// parts combined into the value they represent. If the parts combine to a
324 /// type larger than ValueVT then AssertOp can be used to specify whether the
325 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
326 /// ValueVT (ISD::AssertSext).
328  const SDValue *Parts, unsigned NumParts,
329  MVT PartVT, EVT ValueVT, const Value *V,
330  Optional<CallingConv::ID> CallConv) {
331  assert(ValueVT.isVector() && "Not a vector value");
332  assert(NumParts > 0 && "No parts to assemble!");
333  const bool IsABIRegCopy = CallConv.hasValue();
334 
335  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
336  SDValue Val = Parts[0];
337 
338  // Handle a multi-element vector.
339  if (NumParts > 1) {
340  EVT IntermediateVT;
341  MVT RegisterVT;
342  unsigned NumIntermediates;
343  unsigned NumRegs;
344 
345  if (IsABIRegCopy) {
347  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
348  NumIntermediates, RegisterVT);
349  } else {
350  NumRegs =
351  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
352  NumIntermediates, RegisterVT);
353  }
354 
355  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
356  NumParts = NumRegs; // Silence a compiler warning.
357  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
358  assert(RegisterVT.getSizeInBits() ==
359  Parts[0].getSimpleValueType().getSizeInBits() &&
360  "Part type sizes don't match!");
361 
362  // Assemble the parts into intermediate operands.
363  SmallVector<SDValue, 8> Ops(NumIntermediates);
364  if (NumIntermediates == NumParts) {
365  // If the register was not expanded, truncate or copy the value,
366  // as appropriate.
367  for (unsigned i = 0; i != NumParts; ++i)
368  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
369  PartVT, IntermediateVT, V, CallConv);
370  } else if (NumParts > 0) {
371  // If the intermediate type was expanded, build the intermediate
372  // operands from the parts.
373  assert(NumParts % NumIntermediates == 0 &&
374  "Must expand into a divisible number of parts!");
375  unsigned Factor = NumParts / NumIntermediates;
376  for (unsigned i = 0; i != NumIntermediates; ++i)
377  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
378  PartVT, IntermediateVT, V, CallConv);
379  }
380 
381  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
382  // intermediate operands.
383  EVT BuiltVectorTy =
384  IntermediateVT.isVector()
386  *DAG.getContext(), IntermediateVT.getScalarType(),
387  IntermediateVT.getVectorElementCount() * NumParts)
388  : EVT::getVectorVT(*DAG.getContext(),
389  IntermediateVT.getScalarType(),
390  NumIntermediates);
391  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
393  DL, BuiltVectorTy, Ops);
394  }
395 
396  // There is now one part, held in Val. Correct it to match ValueVT.
397  EVT PartEVT = Val.getValueType();
398 
399  if (PartEVT == ValueVT)
400  return Val;
401 
402  if (PartEVT.isVector()) {
403  // Vector/Vector bitcast.
404  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
405  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
406 
407  // If the element type of the source/dest vectors are the same, but the
408  // parts vector has more elements than the value vector, then we have a
409  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
410  // elements we want.
411  if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
414  (PartEVT.getVectorElementCount().isScalable() ==
415  ValueVT.getVectorElementCount().isScalable()) &&
416  "Cannot narrow, it would be a lossy transformation");
417  PartEVT =
419  ValueVT.getVectorElementCount());
420  Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
421  DAG.getVectorIdxConstant(0, DL));
422  if (PartEVT == ValueVT)
423  return Val;
424  }
425 
426  // Promoted vector extract
427  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
428  }
429 
430  // Trivial bitcast if the types are the same size and the destination
431  // vector type is legal.
432  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
433  TLI.isTypeLegal(ValueVT))
434  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435 
436  if (ValueVT.getVectorNumElements() != 1) {
437  // Certain ABIs require that vectors are passed as integers. For vectors
438  // are the same size, this is an obvious bitcast.
439  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
440  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
441  } else if (ValueVT.bitsLT(PartEVT)) {
442  const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
443  EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
444  // Drop the extra bits.
445  Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
446  return DAG.getBitcast(ValueVT, Val);
447  }
448 
450  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
451  return DAG.getUNDEF(ValueVT);
452  }
453 
454  // Handle cases such as i8 -> <1 x i1>
455  EVT ValueSVT = ValueVT.getVectorElementType();
456  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
457  if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())
458  Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
459  else
460  Val = ValueVT.isFloatingPoint()
461  ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
462  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
463  }
464 
465  return DAG.getBuildVector(ValueVT, DL, Val);
466 }
467 
468 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
469  SDValue Val, SDValue *Parts, unsigned NumParts,
470  MVT PartVT, const Value *V,
471  Optional<CallingConv::ID> CallConv);
472 
473 /// getCopyToParts - Create a series of nodes that contain the specified value
474 /// split into legal parts. If the parts contain more bits than Val, then, for
475 /// integers, ExtendKind can be used to specify how to generate the extra bits.
476 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
477  SDValue *Parts, unsigned NumParts, MVT PartVT,
478  const Value *V,
479  Optional<CallingConv::ID> CallConv = None,
480  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
481  // Let the target split the parts if it wants to
482  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
483  if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
484  CallConv))
485  return;
486  EVT ValueVT = Val.getValueType();
487 
488  // Handle the vector case separately.
489  if (ValueVT.isVector())
490  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
491  CallConv);
492 
493  unsigned PartBits = PartVT.getSizeInBits();
494  unsigned OrigNumParts = NumParts;
495  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
496  "Copying to an illegal type!");
497 
498  if (NumParts == 0)
499  return;
500 
501  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
502  EVT PartEVT = PartVT;
503  if (PartEVT == ValueVT) {
504  assert(NumParts == 1 && "No-op copy with multiple parts!");
505  Parts[0] = Val;
506  return;
507  }
508 
509  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
510  // If the parts cover more bits than the value has, promote the value.
511  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
512  assert(NumParts == 1 && "Do not know what to promote to!");
513  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
514  } else {
515  if (ValueVT.isFloatingPoint()) {
516  // FP values need to be bitcast, then extended if they are being put
517  // into a larger container.
518  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
519  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
520  }
521  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
522  ValueVT.isInteger() &&
523  "Unknown mismatch!");
524  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
525  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
526  if (PartVT == MVT::x86mmx)
527  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
528  }
529  } else if (PartBits == ValueVT.getSizeInBits()) {
530  // Different types of the same size.
531  assert(NumParts == 1 && PartEVT != ValueVT);
532  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
534  // If the parts cover less bits than value has, truncate the value.
535  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
536  ValueVT.isInteger() &&
537  "Unknown mismatch!");
538  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
539  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
540  if (PartVT == MVT::x86mmx)
541  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
542  }
543 
544  // The value may have changed - recompute ValueVT.
545  ValueVT = Val.getValueType();
546  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
547  "Failed to tile the value with PartVT!");
548 
549  if (NumParts == 1) {
550  if (PartEVT != ValueVT) {
552  "scalar-to-vector conversion failed");
553  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
554  }
555 
556  Parts[0] = Val;
557  return;
558  }
559 
560  // Expand the value into multiple parts.
561  if (NumParts & (NumParts - 1)) {
562  // The number of parts is not a power of 2. Split off and copy the tail.
563  assert(PartVT.isInteger() && ValueVT.isInteger() &&
564  "Do not know what to expand to!");
565  unsigned RoundParts = 1 << Log2_32(NumParts);
566  unsigned RoundBits = RoundParts * PartBits;
567  unsigned OddParts = NumParts - RoundParts;
568  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
569  DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
570 
571  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
572  CallConv);
573 
574  if (DAG.getDataLayout().isBigEndian())
575  // The odd parts were reversed by getCopyToParts - unreverse them.
576  std::reverse(Parts + RoundParts, Parts + NumParts);
577 
578  NumParts = RoundParts;
579  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
580  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
581  }
582 
583  // The number of parts is a power of 2. Repeatedly bisect the value using
584  // EXTRACT_ELEMENT.
585  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
587  ValueVT.getSizeInBits()),
588  Val);
589 
590  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
591  for (unsigned i = 0; i < NumParts; i += StepSize) {
592  unsigned ThisBits = StepSize * PartBits / 2;
593  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
594  SDValue &Part0 = Parts[i];
595  SDValue &Part1 = Parts[i+StepSize/2];
596 
597  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
598  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
599  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
600  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
601 
602  if (ThisBits == PartBits && ThisVT != PartVT) {
603  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
604  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
605  }
606  }
607  }
608 
609  if (DAG.getDataLayout().isBigEndian())
610  std::reverse(Parts, Parts + OrigNumParts);
611 }
612 
614  const SDLoc &DL, EVT PartVT) {
615  if (!PartVT.isVector())
616  return SDValue();
617 
618  EVT ValueVT = Val.getValueType();
619  ElementCount PartNumElts = PartVT.getVectorElementCount();
620  ElementCount ValueNumElts = ValueVT.getVectorElementCount();
621 
622  // We only support widening vectors with equivalent element types and
623  // fixed/scalable properties. If a target needs to widen a fixed-length type
624  // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
625  if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
626  PartNumElts.isScalable() != ValueNumElts.isScalable() ||
627  PartVT.getVectorElementType() != ValueVT.getVectorElementType())
628  return SDValue();
629 
630  // Widening a scalable vector to another scalable vector is done by inserting
631  // the vector into a larger undef one.
632  if (PartNumElts.isScalable())
633  return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
634  Val, DAG.getVectorIdxConstant(0, DL));
635 
636  EVT ElementVT = PartVT.getVectorElementType();
637  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
638  // undef elements.
640  DAG.ExtractVectorElements(Val, Ops);
641  SDValue EltUndef = DAG.getUNDEF(ElementVT);
642  Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
643 
644  // FIXME: Use CONCAT for 2x -> 4x.
645  return DAG.getBuildVector(PartVT, DL, Ops);
646 }
647 
648 /// getCopyToPartsVector - Create a series of nodes that contain the specified
649 /// value split into legal parts.
650 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
651  SDValue Val, SDValue *Parts, unsigned NumParts,
652  MVT PartVT, const Value *V,
653  Optional<CallingConv::ID> CallConv) {
654  EVT ValueVT = Val.getValueType();
655  assert(ValueVT.isVector() && "Not a vector");
656  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
657  const bool IsABIRegCopy = CallConv.hasValue();
658 
659  if (NumParts == 1) {
660  EVT PartEVT = PartVT;
661  if (PartEVT == ValueVT) {
662  // Nothing to do.
663  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
664  // Bitconvert vector->vector case.
665  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
666  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
667  Val = Widened;
668  } else if (PartVT.isVector() &&
669  PartEVT.getVectorElementType().bitsGE(
670  ValueVT.getVectorElementType()) &&
671  PartEVT.getVectorElementCount() ==
672  ValueVT.getVectorElementCount()) {
673 
674  // Promoted vector extract
675  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
676  } else {
677  if (ValueVT.getVectorElementCount().isScalar()) {
678  Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
679  DAG.getVectorIdxConstant(0, DL));
680  } else {
681  uint64_t ValueSize = ValueVT.getFixedSizeInBits();
682  assert(PartVT.getFixedSizeInBits() > ValueSize &&
683  "lossy conversion of vector to scalar type");
684  EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
685  Val = DAG.getBitcast(IntermediateType, Val);
686  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
687  }
688  }
689 
690  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
691  Parts[0] = Val;
692  return;
693  }
694 
695  // Handle a multi-element vector.
696  EVT IntermediateVT;
697  MVT RegisterVT;
698  unsigned NumIntermediates;
699  unsigned NumRegs;
700  if (IsABIRegCopy) {
702  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
703  NumIntermediates, RegisterVT);
704  } else {
705  NumRegs =
706  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
707  NumIntermediates, RegisterVT);
708  }
709 
710  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
711  NumParts = NumRegs; // Silence a compiler warning.
712  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
713 
714  assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
715  "Mixing scalable and fixed vectors when copying in parts");
716 
717  Optional<ElementCount> DestEltCnt;
718 
719  if (IntermediateVT.isVector())
720  DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
721  else
722  DestEltCnt = ElementCount::getFixed(NumIntermediates);
723 
724  EVT BuiltVectorTy = EVT::getVectorVT(
725  *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());
726 
727  if (ValueVT == BuiltVectorTy) {
728  // Nothing to do.
729  } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
730  // Bitconvert vector->vector case.
731  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
732  } else {
733  if (BuiltVectorTy.getVectorElementType().bitsGT(
734  ValueVT.getVectorElementType())) {
735  // Integer promotion.
736  ValueVT = EVT::getVectorVT(*DAG.getContext(),
737  BuiltVectorTy.getVectorElementType(),
738  ValueVT.getVectorElementCount());
739  Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
740  }
741 
742  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
743  Val = Widened;
744  }
745  }
746 
747  assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
748 
749  // Split the vector into intermediate operands.
750  SmallVector<SDValue, 8> Ops(NumIntermediates);
751  for (unsigned i = 0; i != NumIntermediates; ++i) {
752  if (IntermediateVT.isVector()) {
753  // This does something sensible for scalable vectors - see the
754  // definition of EXTRACT_SUBVECTOR for further details.
755  unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
756  Ops[i] =
757  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
758  DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
759  } else {
760  Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
761  DAG.getVectorIdxConstant(i, DL));
762  }
763  }
764 
765  // Split the intermediate operands into legal parts.
766  if (NumParts == NumIntermediates) {
767  // If the register was not expanded, promote or copy the value,
768  // as appropriate.
769  for (unsigned i = 0; i != NumParts; ++i)
770  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
771  } else if (NumParts > 0) {
772  // If the intermediate type was expanded, split each the value into
773  // legal parts.
774  assert(NumIntermediates != 0 && "division by zero");
775  assert(NumParts % NumIntermediates == 0 &&
776  "Must expand into a divisible number of parts!");
777  unsigned Factor = NumParts / NumIntermediates;
778  for (unsigned i = 0; i != NumIntermediates; ++i)
779  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
780  CallConv);
781  }
782 }
783 
785  EVT valuevt, Optional<CallingConv::ID> CC)
786  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
787  RegCount(1, regs.size()), CallConv(CC) {}
788 
790  const DataLayout &DL, unsigned Reg, Type *Ty,
792  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
793 
794  CallConv = CC;
795 
796  for (EVT ValueVT : ValueVTs) {
797  unsigned NumRegs =
798  isABIMangled()
799  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
800  : TLI.getNumRegisters(Context, ValueVT);
801  MVT RegisterVT =
802  isABIMangled()
803  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
804  : TLI.getRegisterType(Context, ValueVT);
805  for (unsigned i = 0; i != NumRegs; ++i)
806  Regs.push_back(Reg + i);
807  RegVTs.push_back(RegisterVT);
808  RegCount.push_back(NumRegs);
809  Reg += NumRegs;
810  }
811 }
812 
814  FunctionLoweringInfo &FuncInfo,
815  const SDLoc &dl, SDValue &Chain,
816  SDValue *Flag, const Value *V) const {
817  // A Value with type {} or [0 x %t] needs no registers.
818  if (ValueVTs.empty())
819  return SDValue();
820 
821  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
822 
823  // Assemble the legal parts into the final values.
824  SmallVector<SDValue, 4> Values(ValueVTs.size());
826  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
827  // Copy the legal parts from the registers.
828  EVT ValueVT = ValueVTs[Value];
829  unsigned NumRegs = RegCount[Value];
830  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
831  *DAG.getContext(),
833  : RegVTs[Value];
834 
835  Parts.resize(NumRegs);
836  for (unsigned i = 0; i != NumRegs; ++i) {
837  SDValue P;
838  if (!Flag) {
839  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
840  } else {
841  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
842  *Flag = P.getValue(2);
843  }
844 
845  Chain = P.getValue(1);
846  Parts[i] = P;
847 
848  // If the source register was virtual and if we know something about it,
849  // add an assert node.
850  if (!Register::isVirtualRegister(Regs[Part + i]) ||
851  !RegisterVT.isInteger())
852  continue;
853 
855  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
856  if (!LOI)
857  continue;
858 
859  unsigned RegSize = RegisterVT.getScalarSizeInBits();
860  unsigned NumSignBits = LOI->NumSignBits;
861  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
862 
863  if (NumZeroBits == RegSize) {
864  // The current value is a zero.
865  // Explicitly express that as it would be easier for
866  // optimizations to kick in.
867  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
868  continue;
869  }
870 
871  // FIXME: We capture more information than the dag can represent. For
872  // now, just use the tightest assertzext/assertsext possible.
873  bool isSExt;
874  EVT FromVT(MVT::Other);
875  if (NumZeroBits) {
876  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
877  isSExt = false;
878  } else if (NumSignBits > 1) {
879  FromVT =
880  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
881  isSExt = true;
882  } else {
883  continue;
884  }
885  // Add an assertion node.
886  assert(FromVT != MVT::Other);
887  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
888  RegisterVT, P, DAG.getValueType(FromVT));
889  }
890 
891  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
892  RegisterVT, ValueVT, V, CallConv);
893  Part += NumRegs;
894  Parts.clear();
895  }
896 
897  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
898 }
899 
901  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
902  const Value *V,
903  ISD::NodeType PreferredExtendType) const {
904  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
905  ISD::NodeType ExtendKind = PreferredExtendType;
906 
907  // Get the list of the values's legal parts.
908  unsigned NumRegs = Regs.size();
909  SmallVector<SDValue, 8> Parts(NumRegs);
910  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
911  unsigned NumParts = RegCount[Value];
912 
913  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
914  *DAG.getContext(),
916  : RegVTs[Value];
917 
918  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
919  ExtendKind = ISD::ZERO_EXTEND;
920 
921  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
922  NumParts, RegisterVT, V, CallConv, ExtendKind);
923  Part += NumParts;
924  }
925 
926  // Copy the parts into the registers.
927  SmallVector<SDValue, 8> Chains(NumRegs);
928  for (unsigned i = 0; i != NumRegs; ++i) {
929  SDValue Part;
930  if (!Flag) {
931  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
932  } else {
933  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
934  *Flag = Part.getValue(1);
935  }
936 
937  Chains[i] = Part.getValue(0);
938  }
939 
940  if (NumRegs == 1 || Flag)
941  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
942  // flagged to it. That is the CopyToReg nodes and the user are considered
943  // a single scheduling unit. If we create a TokenFactor and return it as
944  // chain, then the TokenFactor is both a predecessor (operand) of the
945  // user as well as a successor (the TF operands are flagged to the user).
946  // c1, f1 = CopyToReg
947  // c2, f2 = CopyToReg
948  // c3 = TokenFactor c1, c2
949  // ...
950  // = op c3, ..., f2
951  Chain = Chains[NumRegs-1];
952  else
953  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
954 }
955 
956 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
957  unsigned MatchingIdx, const SDLoc &dl,
958  SelectionDAG &DAG,
959  std::vector<SDValue> &Ops) const {
960  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
961 
962  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
963  if (HasMatching)
965  else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
966  // Put the register class of the virtual registers in the flag word. That
967  // way, later passes can recompute register class constraints for inline
968  // assembly as well as normal instructions.
969  // Don't do this for tied operands that can use the regclass information
970  // from the def.
972  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
974  }
975 
976  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
977  Ops.push_back(Res);
978 
979  if (Code == InlineAsm::Kind_Clobber) {
980  // Clobbers should always have a 1:1 mapping with registers, and may
981  // reference registers that have illegal (e.g. vector) types. Hence, we
982  // shouldn't try to apply any sort of splitting logic to them.
983  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
984  "No 1:1 mapping from clobbers to regs?");
986  (void)SP;
987  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
988  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
989  assert(
990  (Regs[I] != SP ||
992  "If we clobbered the stack pointer, MFI should know about it.");
993  }
994  return;
995  }
996 
997  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
998  MVT RegisterVT = RegVTs[Value];
999  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1000  RegisterVT);
1001  for (unsigned i = 0; i != NumRegs; ++i) {
1002  assert(Reg < Regs.size() && "Mismatch in # registers expected");
1003  unsigned TheReg = Regs[Reg++];
1004  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1005  }
1006  }
1007 }
1008 
1012  unsigned I = 0;
1013  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1014  unsigned RegCount = std::get<0>(CountAndVT);
1015  MVT RegisterVT = std::get<1>(CountAndVT);
1016  TypeSize RegisterSize = RegisterVT.getSizeInBits();
1017  for (unsigned E = I + RegCount; I != E; ++I)
1018  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1019  }
1020  return OutVec;
1021 }
1022 
1024  const TargetLibraryInfo *li) {
1025  AA = aa;
1026  GFI = gfi;
1027  LibInfo = li;
1028  DL = &DAG.getDataLayout();
1029  Context = DAG.getContext();
1030  LPadToCallSiteMap.clear();
1031  SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1032 }
1033 
1035  NodeMap.clear();
1036  UnusedArgNodeMap.clear();
1037  PendingLoads.clear();
1038  PendingExports.clear();
1039  PendingConstrainedFP.clear();
1040  PendingConstrainedFPStrict.clear();
1041  CurInst = nullptr;
1042  HasTailCall = false;
1043  SDNodeOrder = LowestSDNodeOrder;
1045 }
1046 
1048  DanglingDebugInfoMap.clear();
1049 }
1050 
1051 // Update DAG root to include dependencies on Pending chains.
1052 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1053  SDValue Root = DAG.getRoot();
1054 
1055  if (Pending.empty())
1056  return Root;
1057 
1058  // Add current root to PendingChains, unless we already indirectly
1059  // depend on it.
1060  if (Root.getOpcode() != ISD::EntryToken) {
1061  unsigned i = 0, e = Pending.size();
1062  for (; i != e; ++i) {
1063  assert(Pending[i].getNode()->getNumOperands() > 1);
1064  if (Pending[i].getNode()->getOperand(0) == Root)
1065  break; // Don't add the root if we already indirectly depend on it.
1066  }
1067 
1068  if (i == e)
1069  Pending.push_back(Root);
1070  }
1071 
1072  if (Pending.size() == 1)
1073  Root = Pending[0];
1074  else
1075  Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1076 
1077  DAG.setRoot(Root);
1078  Pending.clear();
1079  return Root;
1080 }
1081 
1083  return updateRoot(PendingLoads);
1084 }
1085 
1087  // Chain up all pending constrained intrinsics together with all
1088  // pending loads, by simply appending them to PendingLoads and
1089  // then calling getMemoryRoot().
1090  PendingLoads.reserve(PendingLoads.size() +
1091  PendingConstrainedFP.size() +
1092  PendingConstrainedFPStrict.size());
1093  PendingLoads.append(PendingConstrainedFP.begin(),
1094  PendingConstrainedFP.end());
1095  PendingLoads.append(PendingConstrainedFPStrict.begin(),
1096  PendingConstrainedFPStrict.end());
1097  PendingConstrainedFP.clear();
1098  PendingConstrainedFPStrict.clear();
1099  return getMemoryRoot();
1100 }
1101 
1103  // We need to emit pending fpexcept.strict constrained intrinsics,
1104  // so append them to the PendingExports list.
1105  PendingExports.append(PendingConstrainedFPStrict.begin(),
1106  PendingConstrainedFPStrict.end());
1107  PendingConstrainedFPStrict.clear();
1108  return updateRoot(PendingExports);
1109 }
1110 
1112  // Set up outgoing PHI node register values before emitting the terminator.
1113  if (I.isTerminator()) {
1114  HandlePHINodesInSuccessorBlocks(I.getParent());
1115  }
1116 
1117  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1118  if (!isa<DbgInfoIntrinsic>(I))
1119  ++SDNodeOrder;
1120 
1121  CurInst = &I;
1122 
1123  visit(I.getOpcode(), I);
1124 
1125  if (!I.isTerminator() && !HasTailCall &&
1126  !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1128 
1129  CurInst = nullptr;
1130 }
1131 
1132 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1133  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1134 }
1135 
1136 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1137  // Note: this doesn't use InstVisitor, because it has to work with
1138  // ConstantExpr's in addition to instructions.
1139  switch (Opcode) {
1140  default: llvm_unreachable("Unknown instruction type encountered!");
1141  // Build the switch statement using the Instruction.def file.
1142 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1143  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1144 #include "llvm/IR/Instruction.def"
1145  }
1146 }
1147 
1149  DebugLoc DL, unsigned Order) {
1150  // We treat variadic dbg_values differently at this stage.
1151  if (DI->hasArgList()) {
1152  // For variadic dbg_values we will now insert an undef.
1153  // FIXME: We can potentially recover these!
1155  for (const Value *V : DI->getValues()) {
1156  auto Undef = UndefValue::get(V->getType());
1157  Locs.push_back(SDDbgOperand::fromConst(Undef));
1158  }
1160  DI->getVariable(), DI->getExpression(), Locs, {},
1161  /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);
1162  DAG.AddDbgValue(SDV, /*isParameter=*/false);
1163  } else {
1164  // TODO: Dangling debug info will eventually either be resolved or produce
1165  // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1166  // between the original dbg.value location and its resolved DBG_VALUE,
1167  // which we should ideally fill with an extra Undef DBG_VALUE.
1168  assert(DI->getNumVariableLocationOps() == 1 &&
1169  "DbgValueInst without an ArgList should have a single location "
1170  "operand.");
1171  DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);
1172  }
1173 }
1174 
1176  const DIExpression *Expr) {
1177  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1178  const DbgValueInst *DI = DDI.getDI();
1179  DIVariable *DanglingVariable = DI->getVariable();
1180  DIExpression *DanglingExpr = DI->getExpression();
1181  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1182  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1183  return true;
1184  }
1185  return false;
1186  };
1187 
1188  for (auto &DDIMI : DanglingDebugInfoMap) {
1189  DanglingDebugInfoVector &DDIV = DDIMI.second;
1190 
1191  // If debug info is to be dropped, run it through final checks to see
1192  // whether it can be salvaged.
1193  for (auto &DDI : DDIV)
1194  if (isMatchingDbgValue(DDI))
1196 
1197  erase_if(DDIV, isMatchingDbgValue);
1198  }
1199 }
1200 
1201 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1202 // generate the debug data structures now that we've seen its definition.
1204  SDValue Val) {
1205  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1206  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1207  return;
1208 
1209  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1210  for (auto &DDI : DDIV) {
1211  const DbgValueInst *DI = DDI.getDI();
1212  assert(!DI->hasArgList() && "Not implemented for variadic dbg_values");
1213  assert(DI && "Ill-formed DanglingDebugInfo");
1214  DebugLoc dl = DDI.getdl();
1215  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1216  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1217  DILocalVariable *Variable = DI->getVariable();
1218  DIExpression *Expr = DI->getExpression();
1219  assert(Variable->isValidLocationForIntrinsic(dl) &&
1220  "Expected inlined-at fields to agree");
1221  SDDbgValue *SDV;
1222  if (Val.getNode()) {
1223  // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1224  // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1225  // we couldn't resolve it directly when examining the DbgValue intrinsic
1226  // in the first place we should not be more successful here). Unless we
1227  // have some test case that prove this to be correct we should avoid
1228  // calling EmitFuncArgumentDbgValue here.
1229  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1230  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1231  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1232  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1233  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1234  // inserted after the definition of Val when emitting the instructions
1235  // after ISel. An alternative could be to teach
1236  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1237  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1238  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1239  << ValSDNodeOrder << "\n");
1240  SDV = getDbgValue(Val, Variable, Expr, dl,
1241  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1242  DAG.AddDbgValue(SDV, false);
1243  } else
1244  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1245  << "in EmitFuncArgumentDbgValue\n");
1246  } else {
1247  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1248  auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1249  auto SDV =
1250  DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1251  DAG.AddDbgValue(SDV, false);
1252  }
1253  }
1254  DDIV.clear();
1255 }
1256 
1258  // TODO: For the variadic implementation, instead of only checking the fail
1259  // state of `handleDebugValue`, we need know specifically which values were
1260  // invalid, so that we attempt to salvage only those values when processing
1261  // a DIArgList.
1262  assert(!DDI.getDI()->hasArgList() &&
1263  "Not implemented for variadic dbg_values");
1264  Value *V = DDI.getDI()->getValue(0);
1265  DILocalVariable *Var = DDI.getDI()->getVariable();
1266  DIExpression *Expr = DDI.getDI()->getExpression();
1267  DebugLoc DL = DDI.getdl();
1268  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1269  unsigned SDOrder = DDI.getSDNodeOrder();
1270  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1271  // that DW_OP_stack_value is desired.
1272  assert(isa<DbgValueInst>(DDI.getDI()));
1273  bool StackValue = true;
1274 
1275  // Can this Value can be encoded without any further work?
1276  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))
1277  return;
1278 
1279  // Attempt to salvage back through as many instructions as possible. Bail if
1280  // a non-instruction is seen, such as a constant expression or global
1281  // variable. FIXME: Further work could recover those too.
1282  while (isa<Instruction>(V)) {
1283  Instruction &VAsInst = *cast<Instruction>(V);
1284  // Temporary "0", awaiting real implementation.
1286  SmallVector<Value *, 4> AdditionalValues;
1287  V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1288  AdditionalValues);
1289  // If we cannot salvage any further, and haven't yet found a suitable debug
1290  // expression, bail out.
1291  if (!V)
1292  break;
1293 
1294  // TODO: If AdditionalValues isn't empty, then the salvage can only be
1295  // represented with a DBG_VALUE_LIST, so we give up. When we have support
1296  // here for variadic dbg_values, remove that condition.
1297  if (!AdditionalValues.empty())
1298  break;
1299 
1300  // New value and expr now represent this debuginfo.
1301  Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1302 
1303  // Some kind of simplification occurred: check whether the operand of the
1304  // salvaged debug expression can be encoded in this DAG.
1305  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,
1306  /*IsVariadic=*/false)) {
1307  LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "
1308  << DDI.getDI() << "\nBy stripping back to:\n " << V);
1309  return;
1310  }
1311  }
1312 
1313  // This was the final opportunity to salvage this debug information, and it
1314  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1315  // any earlier variable location.
1316  auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1317  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1318  DAG.AddDbgValue(SDV, false);
1319 
1320  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()
1321  << "\n");
1322  LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)
1323  << "\n");
1324 }
1325 
1327  DILocalVariable *Var,
1328  DIExpression *Expr, DebugLoc dl,
1329  DebugLoc InstDL, unsigned Order,
1330  bool IsVariadic) {
1331  if (Values.empty())
1332  return true;
1333  SmallVector<SDDbgOperand> LocationOps;
1334  SmallVector<SDNode *> Dependencies;
1335  for (const Value *V : Values) {
1336  // Constant value.
1337  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1338  isa<ConstantPointerNull>(V)) {
1339  LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1340  continue;
1341  }
1342 
1343  // If the Value is a frame index, we can create a FrameIndex debug value
1344  // without relying on the DAG at all.
1345  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1346  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1347  if (SI != FuncInfo.StaticAllocaMap.end()) {
1348  LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1349  continue;
1350  }
1351  }
1352 
1353  // Do not use getValue() in here; we don't want to generate code at
1354  // this point if it hasn't been done yet.
1355  SDValue N = NodeMap[V];
1356  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1357  N = UnusedArgNodeMap[V];
1358  if (N.getNode()) {
1359  // Only emit func arg dbg value for non-variadic dbg.values for now.
1360  if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1361  return true;
1362  if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1363  // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1364  // describe stack slot locations.
1365  //
1366  // Consider "int x = 0; int *px = &x;". There are two kinds of
1367  // interesting debug values here after optimization:
1368  //
1369  // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1370  // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1371  //
1372  // Both describe the direct values of their associated variables.
1373  Dependencies.push_back(N.getNode());
1374  LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1375  continue;
1376  }
1377  LocationOps.emplace_back(
1378  SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1379  continue;
1380  }
1381 
1382  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1383  // Special rules apply for the first dbg.values of parameter variables in a
1384  // function. Identify them by the fact they reference Argument Values, that
1385  // they're parameters, and they are parameters of the current function. We
1386  // need to let them dangle until they get an SDNode.
1387  bool IsParamOfFunc =
1388  isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();
1389  if (IsParamOfFunc)
1390  return false;
1391 
1392  // The value is not used in this block yet (or it would have an SDNode).
1393  // We still want the value to appear for the user if possible -- if it has
1394  // an associated VReg, we can refer to that instead.
1395  auto VMI = FuncInfo.ValueMap.find(V);
1396  if (VMI != FuncInfo.ValueMap.end()) {
1397  unsigned Reg = VMI->second;
1398  // If this is a PHI node, it may be split up into several MI PHI nodes
1399  // (in FunctionLoweringInfo::set).
1400  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1401  V->getType(), None);
1402  if (RFV.occupiesMultipleRegs()) {
1403  // FIXME: We could potentially support variadic dbg_values here.
1404  if (IsVariadic)
1405  return false;
1406  unsigned Offset = 0;
1407  unsigned BitsToDescribe = 0;
1408  if (auto VarSize = Var->getSizeInBits())
1409  BitsToDescribe = *VarSize;
1410  if (auto Fragment = Expr->getFragmentInfo())
1411  BitsToDescribe = Fragment->SizeInBits;
1412  for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1413  // Bail out if all bits are described already.
1414  if (Offset >= BitsToDescribe)
1415  break;
1416  // TODO: handle scalable vectors.
1417  unsigned RegisterSize = RegAndSize.second;
1418  unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1419  ? BitsToDescribe - Offset
1420  : RegisterSize;
1421  auto FragmentExpr = DIExpression::createFragmentExpression(
1422  Expr, Offset, FragmentSize);
1423  if (!FragmentExpr)
1424  continue;
1426  Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);
1427  DAG.AddDbgValue(SDV, false);
1428  Offset += RegisterSize;
1429  }
1430  return true;
1431  }
1432  // We can use simple vreg locations for variadic dbg_values as well.
1433  LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1434  continue;
1435  }
1436  // We failed to create a SDDbgOperand for V.
1437  return false;
1438  }
1439 
1440  // We have created a SDDbgOperand for each Value in Values.
1441  // Should use Order instead of SDNodeOrder?
1442  assert(!LocationOps.empty());
1443  SDDbgValue *SDV =
1444  DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1445  /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);
1446  DAG.AddDbgValue(SDV, /*isParameter=*/false);
1447  return true;
1448 }
1449 
1451  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1452  for (auto &Pair : DanglingDebugInfoMap)
1453  for (auto &DDI : Pair.second)
1456 }
1457 
1458 /// getCopyFromRegs - If there was virtual register allocated for the value V
1459 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1462  SDValue Result;
1463 
1464  if (It != FuncInfo.ValueMap.end()) {
1465  Register InReg = It->second;
1466 
1468  DAG.getDataLayout(), InReg, Ty,
1469  None); // This is not an ABI copy.
1470  SDValue Chain = DAG.getEntryNode();
1471  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1472  V);
1473  resolveDanglingDebugInfo(V, Result);
1474  }
1475 
1476  return Result;
1477 }
1478 
1479 /// getValue - Return an SDValue for the given Value.
1481  // If we already have an SDValue for this value, use it. It's important
1482  // to do this first, so that we don't create a CopyFromReg if we already
1483  // have a regular SDValue.
1484  SDValue &N = NodeMap[V];
1485  if (N.getNode()) return N;
1486 
1487  // If there's a virtual register allocated and initialized for this
1488  // value, use it.
1489  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1490  return copyFromReg;
1491 
1492  // Otherwise create a new SDValue and remember it.
1493  SDValue Val = getValueImpl(V);
1494  NodeMap[V] = Val;
1495  resolveDanglingDebugInfo(V, Val);
1496  return Val;
1497 }
1498 
1499 /// getNonRegisterValue - Return an SDValue for the given Value, but
1500 /// don't look in FuncInfo.ValueMap for a virtual register.
1502  // If we already have an SDValue for this value, use it.
1503  SDValue &N = NodeMap[V];
1504  if (N.getNode()) {
1505  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1506  // Remove the debug location from the node as the node is about to be used
1507  // in a location which may differ from the original debug location. This
1508  // is relevant to Constant and ConstantFP nodes because they can appear
1509  // as constant expressions inside PHI nodes.
1510  N->setDebugLoc(DebugLoc());
1511  }
1512  return N;
1513  }
1514 
1515  // Otherwise create a new SDValue and remember it.
1516  SDValue Val = getValueImpl(V);
1517  NodeMap[V] = Val;
1518  resolveDanglingDebugInfo(V, Val);
1519  return Val;
1520 }
1521 
1522 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1523 /// Create an SDValue for the given value.
1525  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1526 
1527  if (const Constant *C = dyn_cast<Constant>(V)) {
1528  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1529 
1530  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1531  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1532 
1533  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1534  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1535 
1536  if (isa<ConstantPointerNull>(C)) {
1537  unsigned AS = V->getType()->getPointerAddressSpace();
1538  return DAG.getConstant(0, getCurSDLoc(),
1539  TLI.getPointerTy(DAG.getDataLayout(), AS));
1540  }
1541 
1542  if (match(C, m_VScale(DAG.getDataLayout())))
1543  return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1544 
1545  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1546  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1547 
1548  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1549  return DAG.getUNDEF(VT);
1550 
1551  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1552  visit(CE->getOpcode(), *CE);
1553  SDValue N1 = NodeMap[V];
1554  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1555  return N1;
1556  }
1557 
1558  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1560  for (const Use &U : C->operands()) {
1561  SDNode *Val = getValue(U).getNode();
1562  // If the operand is an empty aggregate, there are no values.
1563  if (!Val) continue;
1564  // Add each leaf value from the operand to the Constants list
1565  // to form a flattened list of all the values.
1566  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1567  Constants.push_back(SDValue(Val, i));
1568  }
1569 
1571  }
1572 
1573  if (const ConstantDataSequential *CDS =
1574  dyn_cast<ConstantDataSequential>(C)) {
1576  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1577  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1578  // Add each leaf value from the operand to the Constants list
1579  // to form a flattened list of all the values.
1580  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1581  Ops.push_back(SDValue(Val, i));
1582  }
1583 
1584  if (isa<ArrayType>(CDS->getType()))
1585  return DAG.getMergeValues(Ops, getCurSDLoc());
1586  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1587  }
1588 
1589  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1590  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1591  "Unknown struct or array constant!");
1592 
1593  SmallVector<EVT, 4> ValueVTs;
1594  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1595  unsigned NumElts = ValueVTs.size();
1596  if (NumElts == 0)
1597  return SDValue(); // empty struct
1599  for (unsigned i = 0; i != NumElts; ++i) {
1600  EVT EltVT = ValueVTs[i];
1601  if (isa<UndefValue>(C))
1602  Constants[i] = DAG.getUNDEF(EltVT);
1603  else if (EltVT.isFloatingPoint())
1604  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1605  else
1606  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1607  }
1608 
1610  }
1611 
1612  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1613  return DAG.getBlockAddress(BA, VT);
1614 
1615  if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1616  return getValue(Equiv->getGlobalValue());
1617 
1618  VectorType *VecTy = cast<VectorType>(V->getType());
1619 
1620  // Now that we know the number and type of the elements, get that number of
1621  // elements into the Ops array based on what kind of constant it is.
1622  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1624  unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1625  for (unsigned i = 0; i != NumElements; ++i)
1626  Ops.push_back(getValue(CV->getOperand(i)));
1627 
1628  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1629  } else if (isa<ConstantAggregateZero>(C)) {
1630  EVT EltVT =
1631  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1632 
1633  SDValue Op;
1634  if (EltVT.isFloatingPoint())
1635  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1636  else
1637  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1638 
1639  if (isa<ScalableVectorType>(VecTy))
1640  return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);
1641  else {
1643  Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);
1644  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1645  }
1646  }
1647  llvm_unreachable("Unknown vector constant");
1648  }
1649 
1650  // If this is a static alloca, generate it as the frameindex instead of
1651  // computation.
1652  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1654  FuncInfo.StaticAllocaMap.find(AI);
1655  if (SI != FuncInfo.StaticAllocaMap.end())
1656  return DAG.getFrameIndex(SI->second,
1658  }
1659 
1660  // If this is an instruction which fast-isel has deferred, select it now.
1661  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1662  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1663 
1664  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1665  Inst->getType(), None);
1666  SDValue Chain = DAG.getEntryNode();
1667  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1668  }
1669 
1670  if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {
1671  return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1672  }
1673  llvm_unreachable("Can't get register for value!");
1674 }
1675 
1676 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1678  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1679  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1680  bool IsSEH = isAsynchronousEHPersonality(Pers);
1681  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1682  if (!IsSEH)
1683  CatchPadMBB->setIsEHScopeEntry();
1684  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1685  if (IsMSVCCXX || IsCoreCLR)
1686  CatchPadMBB->setIsEHFuncletEntry();
1687 }
1688 
1689 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1690  // Update machine-CFG edge.
1691  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1692  FuncInfo.MBB->addSuccessor(TargetMBB);
1693  TargetMBB->setIsEHCatchretTarget(true);
1695 
1697  bool IsSEH = isAsynchronousEHPersonality(Pers);
1698  if (IsSEH) {
1699  // If this is not a fall-through branch or optimizations are switched off,
1700  // emit the branch.
1701  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1702  TM.getOptLevel() == CodeGenOpt::None)
1704  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1705  return;
1706  }
1707 
1708  // Figure out the funclet membership for the catchret's successor.
1709  // This will be used by the FuncletLayout pass to determine how to order the
1710  // BB's.
1711  // A 'catchret' returns to the outer scope's color.
1712  Value *ParentPad = I.getCatchSwitchParentPad();
1713  const BasicBlock *SuccessorColor;
1714  if (isa<ConstantTokenNone>(ParentPad))
1715  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1716  else
1717  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1718  assert(SuccessorColor && "No parent funclet for catchret!");
1719  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1720  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1721 
1722  // Create the terminator node.
1724  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1725  DAG.getBasicBlock(SuccessorColorMBB));
1726  DAG.setRoot(Ret);
1727 }
1728 
1729 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1730  // Don't emit any special code for the cleanuppad instruction. It just marks
1731  // the start of an EH scope/funclet.
1734  if (Pers != EHPersonality::Wasm_CXX) {
1737  }
1738 }
1739 
1740 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1741 // not match, it is OK to add only the first unwind destination catchpad to the
1742 // successors, because there will be at least one invoke instruction within the
1743 // catch scope that points to the next unwind destination, if one exists, so
1744 // CFGSort cannot mess up with BB sorting order.
1745 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1746 // call within them, and catchpads only consisting of 'catch (...)' have a
1747 // '__cxa_end_catch' call within them, both of which generate invokes in case
1748 // the next unwind destination exists, i.e., the next unwind destination is not
1749 // the caller.)
1750 //
1751 // Having at most one EH pad successor is also simpler and helps later
1752 // transformations.
1753 //
1754 // For example,
1755 // current:
1756 // invoke void @foo to ... unwind label %catch.dispatch
1757 // catch.dispatch:
1758 // %0 = catchswitch within ... [label %catch.start] unwind label %next
1759 // catch.start:
1760 // ...
1761 // ... in this BB or some other child BB dominated by this BB there will be an
1762 // invoke that points to 'next' BB as an unwind destination
1763 //
1764 // next: ; We don't need to add this to 'current' BB's successor
1765 // ...
1767  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1768  BranchProbability Prob,
1769  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1770  &UnwindDests) {
1771  while (EHPadBB) {
1772  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1773  if (isa<CleanupPadInst>(Pad)) {
1774  // Stop on cleanup pads.
1775  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1776  UnwindDests.back().first->setIsEHScopeEntry();
1777  break;
1778  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1779  // Add the catchpad handlers to the possible destinations. We don't
1780  // continue to the unwind destination of the catchswitch for wasm.
1781  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1782  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1783  UnwindDests.back().first->setIsEHScopeEntry();
1784  }
1785  break;
1786  } else {
1787  continue;
1788  }
1789  }
1790 }
1791 
1792 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1793 /// many places it could ultimately go. In the IR, we have a single unwind
1794 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1795 /// This function skips over imaginary basic blocks that hold catchswitch
1796 /// instructions, and finds all the "real" machine
1797 /// basic block destinations. As those destinations may not be successors of
1798 /// EHPadBB, here we also calculate the edge probability to those destinations.
1799 /// The passed-in Prob is the edge probability to EHPadBB.
1801  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1802  BranchProbability Prob,
1803  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1804  &UnwindDests) {
1805  EHPersonality Personality =
1807  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1808  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1809  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1810  bool IsSEH = isAsynchronousEHPersonality(Personality);
1811 
1812  if (IsWasmCXX) {
1813  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1814  assert(UnwindDests.size() <= 1 &&
1815  "There should be at most one unwind destination for wasm");
1816  return;
1817  }
1818 
1819  while (EHPadBB) {
1820  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1821  BasicBlock *NewEHPadBB = nullptr;
1822  if (isa<LandingPadInst>(Pad)) {
1823  // Stop on landingpads. They are not funclets.
1824  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1825  break;
1826  } else if (isa<CleanupPadInst>(Pad)) {
1827  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1828  // personalities.
1829  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1830  UnwindDests.back().first->setIsEHScopeEntry();
1831  UnwindDests.back().first->setIsEHFuncletEntry();
1832  break;
1833  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1834  // Add the catchpad handlers to the possible destinations.
1835  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1836  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1837  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1838  if (IsMSVCCXX || IsCoreCLR)
1839  UnwindDests.back().first->setIsEHFuncletEntry();
1840  if (!IsSEH)
1841  UnwindDests.back().first->setIsEHScopeEntry();
1842  }
1843  NewEHPadBB = CatchSwitch->getUnwindDest();
1844  } else {
1845  continue;
1846  }
1847 
1848  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1849  if (BPI && NewEHPadBB)
1850  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1851  EHPadBB = NewEHPadBB;
1852  }
1853 }
1854 
1855 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1856  // Update successor info.
1858  auto UnwindDest = I.getUnwindDest();
1860  BranchProbability UnwindDestProb =
1861  (BPI && UnwindDest)
1862  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1864  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1865  for (auto &UnwindDest : UnwindDests) {
1866  UnwindDest.first->setIsEHPad();
1867  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1868  }
1870 
1871  // Create the terminator node.
1872  SDValue Ret =
1874  DAG.setRoot(Ret);
1875 }
1876 
1877 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1878  report_fatal_error("visitCatchSwitch not yet implemented!");
1879 }
1880 
1881 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1882  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1883  auto &DL = DAG.getDataLayout();
1884  SDValue Chain = getControlRoot();
1886  SmallVector<SDValue, 8> OutVals;
1887 
1888  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1889  // lower
1890  //
1891  // %val = call <ty> @llvm.experimental.deoptimize()
1892  // ret <ty> %val
1893  //
1894  // differently.
1895  if (I.getParent()->getTerminatingDeoptimizeCall()) {
1897  return;
1898  }
1899 
1900  if (!FuncInfo.CanLowerReturn) {
1901  unsigned DemoteReg = FuncInfo.DemoteRegister;
1902  const Function *F = I.getParent()->getParent();
1903 
1904  // Emit a store of the return value through the virtual register.
1905  // Leave Outs empty so that LowerReturn won't try to load return
1906  // registers the usual way.
1907  SmallVector<EVT, 1> PtrValueVTs;
1908  ComputeValueVTs(TLI, DL,
1909  F->getReturnType()->getPointerTo(
1911  PtrValueVTs);
1912 
1914  DemoteReg, PtrValueVTs[0]);
1915  SDValue RetOp = getValue(I.getOperand(0));
1916 
1917  SmallVector<EVT, 4> ValueVTs, MemVTs;
1919  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1920  &Offsets);
1921  unsigned NumValues = ValueVTs.size();
1922 
1923  SmallVector<SDValue, 4> Chains(NumValues);
1924  Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1925  for (unsigned i = 0; i != NumValues; ++i) {
1926  // An aggregate return value cannot wrap around the address space, so
1927  // offsets to its parts don't wrap either.
1928  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
1930 
1931  SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1932  if (MemVTs[i] != ValueVTs[i])
1933  Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1934  Chains[i] = DAG.getStore(
1935  Chain, getCurSDLoc(), Val,
1936  // FIXME: better loc info would be nice.
1938  commonAlignment(BaseAlign, Offsets[i]));
1939  }
1940 
1942  MVT::Other, Chains);
1943  } else if (I.getNumOperands() != 0) {
1944  SmallVector<EVT, 4> ValueVTs;
1945  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1946  unsigned NumValues = ValueVTs.size();
1947  if (NumValues) {
1948  SDValue RetOp = getValue(I.getOperand(0));
1949 
1950  const Function *F = I.getParent()->getParent();
1951 
1952  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1953  I.getOperand(0)->getType(), F->getCallingConv(),
1954  /*IsVarArg*/ false, DL);
1955 
1956  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1957  if (F->getAttributes().hasRetAttr(Attribute::SExt))
1958  ExtendKind = ISD::SIGN_EXTEND;
1959  else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
1960  ExtendKind = ISD::ZERO_EXTEND;
1961 
1962  LLVMContext &Context = F->getContext();
1963  bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
1964 
1965  for (unsigned j = 0; j != NumValues; ++j) {
1966  EVT VT = ValueVTs[j];
1967 
1968  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1969  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1970 
1971  CallingConv::ID CC = F->getCallingConv();
1972 
1973  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1974  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1975  SmallVector<SDValue, 4> Parts(NumParts);
1977  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1978  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1979 
1980  // 'inreg' on function refers to return value
1981  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1982  if (RetInReg)
1983  Flags.setInReg();
1984 
1985  if (I.getOperand(0)->getType()->isPointerTy()) {
1986  Flags.setPointer();
1987  Flags.setPointerAddrSpace(
1988  cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
1989  }
1990 
1991  if (NeedsRegBlock) {
1992  Flags.setInConsecutiveRegs();
1993  if (j == NumValues - 1)
1994  Flags.setInConsecutiveRegsLast();
1995  }
1996 
1997  // Propagate extension type if any
1998  if (ExtendKind == ISD::SIGN_EXTEND)
1999  Flags.setSExt();
2000  else if (ExtendKind == ISD::ZERO_EXTEND)
2001  Flags.setZExt();
2002 
2003  for (unsigned i = 0; i < NumParts; ++i) {
2004  Outs.push_back(ISD::OutputArg(Flags,
2005  Parts[i].getValueType().getSimpleVT(),
2006  VT, /*isfixed=*/true, 0, 0));
2007  OutVals.push_back(Parts[i]);
2008  }
2009  }
2010  }
2011  }
2012 
2013  // Push in swifterror virtual register as the last element of Outs. This makes
2014  // sure swifterror virtual register will be returned in the swifterror
2015  // physical register.
2016  const Function *F = I.getParent()->getParent();
2017  if (TLI.supportSwiftError() &&
2018  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2019  assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2020  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2021  Flags.setSwiftError();
2022  Outs.push_back(ISD::OutputArg(
2023  Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2024  /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2025  // Create SDNode for the swifterror virtual register.
2026  OutVals.push_back(
2029  EVT(TLI.getPointerTy(DL))));
2030  }
2031 
2032  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2033  CallingConv::ID CallConv =
2036  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2037 
2038  // Verify that the target's LowerReturn behaved as expected.
2039  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2040  "LowerReturn didn't return a valid chain!");
2041 
2042  // Update the DAG with the new chain value resulting from return lowering.
2043  DAG.setRoot(Chain);
2044 }
2045 
2046 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2047 /// created for it, emit nodes to copy the value into the virtual
2048 /// registers.
2050  // Skip empty types
2051  if (V->getType()->isEmptyTy())
2052  return;
2053 
2055  if (VMI != FuncInfo.ValueMap.end()) {
2056  assert(!V->use_empty() && "Unused value assigned virtual registers!");
2057  CopyValueToVirtualRegister(V, VMI->second);
2058  }
2059 }
2060 
2061 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2062 /// the current basic block, add it to ValueMap now so that we'll get a
2063 /// CopyTo/FromReg.
2065  // No need to export constants.
2066  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2067 
2068  // Already exported?
2069  if (FuncInfo.isExportedInst(V)) return;
2070 
2071  unsigned Reg = FuncInfo.InitializeRegForValue(V);
2073 }
2074 
2076  const BasicBlock *FromBB) {
2077  // The operands of the setcc have to be in this block. We don't know
2078  // how to export them from some other block.
2079  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2080  // Can export from current BB.
2081  if (VI->getParent() == FromBB)
2082  return true;
2083 
2084  // Is already exported, noop.
2085  return FuncInfo.isExportedInst(V);
2086  }
2087 
2088  // If this is an argument, we can export it if the BB is the entry block or
2089  // if it is already exported.
2090  if (isa<Argument>(V)) {
2091  if (FromBB->isEntryBlock())
2092  return true;
2093 
2094  // Otherwise, can only export this if it is already exported.
2095  return FuncInfo.isExportedInst(V);
2096  }
2097 
2098  // Otherwise, constants can always be exported.
2099  return true;
2100 }
2101 
2102 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2104 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2105  const MachineBasicBlock *Dst) const {
2107  const BasicBlock *SrcBB = Src->getBasicBlock();
2108  const BasicBlock *DstBB = Dst->getBasicBlock();
2109  if (!BPI) {
2110  // If BPI is not available, set the default probability as 1 / N, where N is
2111  // the number of successors.
2112  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2113  return BranchProbability(1, SuccSize);
2114  }
2115  return BPI->getEdgeProbability(SrcBB, DstBB);
2116 }
2117 
2118 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2119  MachineBasicBlock *Dst,
2120  BranchProbability Prob) {
2121  if (!FuncInfo.BPI)
2122  Src->addSuccessorWithoutProb(Dst);
2123  else {
2124  if (Prob.isUnknown())
2125  Prob = getEdgeProbability(Src, Dst);
2126  Src->addSuccessor(Dst, Prob);
2127  }
2128 }
2129 
2130 static bool InBlock(const Value *V, const BasicBlock *BB) {
2131  if (const Instruction *I = dyn_cast<Instruction>(V))
2132  return I->getParent() == BB;
2133  return true;
2134 }
2135 
2136 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2137 /// This function emits a branch and is used at the leaves of an OR or an
2138 /// AND operator tree.
2139 void
2141  MachineBasicBlock *TBB,
2142  MachineBasicBlock *FBB,
2143  MachineBasicBlock *CurBB,
2144  MachineBasicBlock *SwitchBB,
2145  BranchProbability TProb,
2146  BranchProbability FProb,
2147  bool InvertCond) {
2148  const BasicBlock *BB = CurBB->getBasicBlock();
2149 
2150  // If the leaf of the tree is a comparison, merge the condition into
2151  // the caseblock.
2152  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2153  // The operands of the cmp have to be in this block. We don't know
2154  // how to export them from some other block. If this is the first block
2155  // of the sequence, no exporting is needed.
2156  if (CurBB == SwitchBB ||
2157  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2158  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2159  ISD::CondCode Condition;
2160  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2161  ICmpInst::Predicate Pred =
2162  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2163  Condition = getICmpCondCode(Pred);
2164  } else {
2165  const FCmpInst *FC = cast<FCmpInst>(Cond);
2166  FCmpInst::Predicate Pred =
2167  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2168  Condition = getFCmpCondCode(Pred);
2169  if (TM.Options.NoNaNsFPMath)
2170  Condition = getFCmpCodeWithoutNaN(Condition);
2171  }
2172 
2173  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2174  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2175  SL->SwitchCases.push_back(CB);
2176  return;
2177  }
2178  }
2179 
2180  // Create a CaseBlock record representing this branch.
2181  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2182  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2183  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2184  SL->SwitchCases.push_back(CB);
2185 }
2186 
2188  MachineBasicBlock *TBB,
2189  MachineBasicBlock *FBB,
2190  MachineBasicBlock *CurBB,
2191  MachineBasicBlock *SwitchBB,
2193  BranchProbability TProb,
2194  BranchProbability FProb,
2195  bool InvertCond) {
2196  // Skip over not part of the tree and remember to invert op and operands at
2197  // next level.
2198  Value *NotCond;
2199  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2200  InBlock(NotCond, CurBB->getBasicBlock())) {
2201  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2202  !InvertCond);
2203  return;
2204  }
2205 
2206  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2207  const Value *BOpOp0, *BOpOp1;
2208  // Compute the effective opcode for Cond, taking into account whether it needs
2209  // to be inverted, e.g.
2210  // and (not (or A, B)), C
2211  // gets lowered as
2212  // and (and (not A, not B), C)
2214  if (BOp) {
2215  BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2216  ? Instruction::And
2217  : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2218  ? Instruction::Or
2219  : (Instruction::BinaryOps)0);
2220  if (InvertCond) {
2221  if (BOpc == Instruction::And)
2222  BOpc = Instruction::Or;
2223  else if (BOpc == Instruction::Or)
2224  BOpc = Instruction::And;
2225  }
2226  }
2227 
2228  // If this node is not part of the or/and tree, emit it as a branch.
2229  // Note that all nodes in the tree should have same opcode.
2230  bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2231  if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2232  !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2233  !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2234  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2235  TProb, FProb, InvertCond);
2236  return;
2237  }
2238 
2239  // Create TmpBB after CurBB.
2240  MachineFunction::iterator BBI(CurBB);
2243  CurBB->getParent()->insert(++BBI, TmpBB);
2244 
2245  if (Opc == Instruction::Or) {
2246  // Codegen X | Y as:
2247  // BB1:
2248  // jmp_if_X TBB
2249  // jmp TmpBB
2250  // TmpBB:
2251  // jmp_if_Y TBB
2252  // jmp FBB
2253  //
2254 
2255  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2256  // The requirement is that
2257  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2258  // = TrueProb for original BB.
2259  // Assuming the original probabilities are A and B, one choice is to set
2260  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2261  // A/(1+B) and 2B/(1+B). This choice assumes that
2262  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2263  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2264  // TmpBB, but the math is more complicated.
2265 
2266  auto NewTrueProb = TProb / 2;
2267  auto NewFalseProb = TProb / 2 + FProb;
2268  // Emit the LHS condition.
2269  FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2270  NewFalseProb, InvertCond);
2271 
2272  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2273  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2274  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2275  // Emit the RHS condition into TmpBB.
2276  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2277  Probs[1], InvertCond);
2278  } else {
2279  assert(Opc == Instruction::And && "Unknown merge op!");
2280  // Codegen X & Y as:
2281  // BB1:
2282  // jmp_if_X TmpBB
2283  // jmp FBB
2284  // TmpBB:
2285  // jmp_if_Y TBB
2286  // jmp FBB
2287  //
2288  // This requires creation of TmpBB after CurBB.
2289 
2290  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2291  // The requirement is that
2292  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2293  // = FalseProb for original BB.
2294  // Assuming the original probabilities are A and B, one choice is to set
2295  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2296  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2297  // TrueProb for BB1 * FalseProb for TmpBB.
2298 
2299  auto NewTrueProb = TProb + FProb / 2;
2300  auto NewFalseProb = FProb / 2;
2301  // Emit the LHS condition.
2302  FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2303  NewFalseProb, InvertCond);
2304 
2305  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2306  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2307  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2308  // Emit the RHS condition into TmpBB.
2309  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2310  Probs[1], InvertCond);
2311  }
2312 }
2313 
2314 /// If the set of cases should be emitted as a series of branches, return true.
2315 /// If we should emit this as a bunch of and/or'd together conditions, return
2316 /// false.
2317 bool
2318 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2319  if (Cases.size() != 2) return true;
2320 
2321  // If this is two comparisons of the same values or'd or and'd together, they
2322  // will get folded into a single comparison, so don't emit two blocks.
2323  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2324  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2325  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2326  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2327  return false;
2328  }
2329 
2330  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2331  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2332  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2333  Cases[0].CC == Cases[1].CC &&
2334  isa<Constant>(Cases[0].CmpRHS) &&
2335  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2336  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2337  return false;
2338  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2339  return false;
2340  }
2341 
2342  return true;
2343 }
2344 
2345 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2346  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2347 
2348  // Update machine-CFG edges.
2349  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2350 
2351  if (I.isUnconditional()) {
2352  // Update machine-CFG edges.
2353  BrMBB->addSuccessor(Succ0MBB);
2354 
2355  // If this is not a fall-through branch or optimizations are switched off,
2356  // emit the branch.
2357  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2360  DAG.getBasicBlock(Succ0MBB)));
2361 
2362  return;
2363  }
2364 
2365  // If this condition is one of the special cases we handle, do special stuff
2366  // now.
2367  const Value *CondVal = I.getCondition();
2368  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2369 
2370  // If this is a series of conditions that are or'd or and'd together, emit
2371  // this as a sequence of branches instead of setcc's with and/or operations.
2372  // As long as jumps are not expensive (exceptions for multi-use logic ops,
2373  // unpredictable branches, and vector extracts because those jumps are likely
2374  // expensive for any target), this should improve performance.
2375  // For example, instead of something like:
2376  // cmp A, B
2377  // C = seteq
2378  // cmp D, E
2379  // F = setle
2380  // or C, F
2381  // jnz foo
2382  // Emit:
2383  // cmp A, B
2384  // je foo
2385  // cmp D, E
2386  // jle foo
2387  const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2388  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2389  BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2390  Value *Vec;
2391  const Value *BOp0, *BOp1;
2393  if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2394  Opcode = Instruction::And;
2395  else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2396  Opcode = Instruction::Or;
2397 
2398  if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2399  match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2400  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2401  getEdgeProbability(BrMBB, Succ0MBB),
2402  getEdgeProbability(BrMBB, Succ1MBB),
2403  /*InvertCond=*/false);
2404  // If the compares in later blocks need to use values not currently
2405  // exported from this block, export them now. This block should always
2406  // be the first entry.
2407  assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2408 
2409  // Allow some cases to be rejected.
2410  if (ShouldEmitAsBranches(SL->SwitchCases)) {
2411  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2412  ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2413  ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2414  }
2415 
2416  // Emit the branch for this block.
2417  visitSwitchCase(SL->SwitchCases[0], BrMBB);
2418  SL->SwitchCases.erase(SL->SwitchCases.begin());
2419  return;
2420  }
2421 
2422  // Okay, we decided not to do this, remove any inserted MBB's and clear
2423  // SwitchCases.
2424  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2425  FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2426 
2427  SL->SwitchCases.clear();
2428  }
2429  }
2430 
2431  // Create a CaseBlock record representing this branch.
2432  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2433  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2434 
2435  // Use visitSwitchCase to actually insert the fast branch sequence for this
2436  // cond branch.
2437  visitSwitchCase(CB, BrMBB);
2438 }
2439 
2440 /// visitSwitchCase - Emits the necessary code to represent a single node in
2441 /// the binary search tree resulting from lowering a switch instruction.
2443  MachineBasicBlock *SwitchBB) {
2444  SDValue Cond;
2445  SDValue CondLHS = getValue(CB.CmpLHS);
2446  SDLoc dl = CB.DL;
2447 
2448  if (CB.CC == ISD::SETTRUE) {
2449  // Branch or fall through to TrueBB.
2450  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2451  SwitchBB->normalizeSuccProbs();
2452  if (CB.TrueBB != NextBlock(SwitchBB)) {
2454  DAG.getBasicBlock(CB.TrueBB)));
2455  }
2456  return;
2457  }
2458 
2459  auto &TLI = DAG.getTargetLoweringInfo();
2460  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2461 
2462  // Build the setcc now.
2463  if (!CB.CmpMHS) {
2464  // Fold "(X == true)" to X and "(X == false)" to !X to
2465  // handle common cases produced by branch lowering.
2466  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2467  CB.CC == ISD::SETEQ)
2468  Cond = CondLHS;
2469  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2470  CB.CC == ISD::SETEQ) {
2471  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2472  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2473  } else {
2474  SDValue CondRHS = getValue(CB.CmpRHS);
2475 
2476  // If a pointer's DAG type is larger than its memory type then the DAG
2477  // values are zero-extended. This breaks signed comparisons so truncate
2478  // back to the underlying type before doing the compare.
2479  if (CondLHS.getValueType() != MemVT) {
2480  CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2481  CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2482  }
2483  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2484  }
2485  } else {
2486  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2487 
2488  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2489  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2490 
2491  SDValue CmpOp = getValue(CB.CmpMHS);
2492  EVT VT = CmpOp.getValueType();
2493 
2494  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2495  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2496  ISD::SETLE);
2497  } else {
2498  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2499  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2500  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2501  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2502  }
2503  }
2504 
2505  // Update successor info
2506  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2507  // TrueBB and FalseBB are always different unless the incoming IR is
2508  // degenerate. This only happens when running llc on weird IR.
2509  if (CB.TrueBB != CB.FalseBB)
2510  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2511  SwitchBB->normalizeSuccProbs();
2512 
2513  // If the lhs block is the next block, invert the condition so that we can
2514  // fall through to the lhs instead of the rhs block.
2515  if (CB.TrueBB == NextBlock(SwitchBB)) {
2516  std::swap(CB.TrueBB, CB.FalseBB);
2517  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2518  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2519  }
2520 
2521  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2523  DAG.getBasicBlock(CB.TrueBB));
2524 
2525  // Insert the false branch. Do this even if it's a fall through branch,
2526  // this makes it easier to do DAG optimizations which require inverting
2527  // the branch condition.
2528  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2529  DAG.getBasicBlock(CB.FalseBB));
2530 
2531  DAG.setRoot(BrCond);
2532 }
2533 
2534 /// visitJumpTable - Emit JumpTable node in the current MBB
2536  // Emit the code for the jump table
2537  assert(JT.Reg != -1U && "Should lower JT Header first!");
2540  JT.Reg, PTy);
2541  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2542  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2543  MVT::Other, Index.getValue(1),
2544  Table, Index);
2545  DAG.setRoot(BrJumpTable);
2546 }
2547 
2548 /// visitJumpTableHeader - This function emits necessary code to produce index
2549 /// in the JumpTable from switch case.
2551  JumpTableHeader &JTH,
2552  MachineBasicBlock *SwitchBB) {
2553  SDLoc dl = getCurSDLoc();
2554 
2555  // Subtract the lowest switch case value from the value being switched on.
2556  SDValue SwitchOp = getValue(JTH.SValue);
2557  EVT VT = SwitchOp.getValueType();
2558  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2559  DAG.getConstant(JTH.First, dl, VT));
2560 
2561  // The SDNode we just created, which holds the value being switched on minus
2562  // the smallest case value, needs to be copied to a virtual register so it
2563  // can be used as an index into the jump table in a subsequent basic block.
2564  // This value may be smaller or larger than the target's pointer type, and
2565  // therefore require extension or truncating.
2566  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2567  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2568 
2569  unsigned JumpTableReg =
2571  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2572  JumpTableReg, SwitchOp);
2573  JT.Reg = JumpTableReg;
2574 
2575  if (!JTH.FallthroughUnreachable) {
2576  // Emit the range check for the jump table, and branch to the default block
2577  // for the switch statement if the value being switched on exceeds the
2578  // largest case in the switch.
2579  SDValue CMP = DAG.getSetCC(
2581  Sub.getValueType()),
2582  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2583 
2584  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2585  MVT::Other, CopyTo, CMP,
2586  DAG.getBasicBlock(JT.Default));
2587 
2588  // Avoid emitting unnecessary branches to the next block.
2589  if (JT.MBB != NextBlock(SwitchBB))
2590  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2591  DAG.getBasicBlock(JT.MBB));
2592 
2593  DAG.setRoot(BrCond);
2594  } else {
2595  // Avoid emitting unnecessary branches to the next block.
2596  if (JT.MBB != NextBlock(SwitchBB))
2597  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2598  DAG.getBasicBlock(JT.MBB)));
2599  else
2600  DAG.setRoot(CopyTo);
2601  }
2602 }
2603 
2604 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2605 /// variable if there exists one.
2607  SDValue &Chain) {
2608  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2609  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2610  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2611  MachineFunction &MF = DAG.getMachineFunction();
2612  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2613  MachineSDNode *Node =
2614  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2615  if (Global) {
2616  MachinePointerInfo MPInfo(Global);
2620  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2621  DAG.setNodeMemRefs(Node, {MemRef});
2622  }
2623  if (PtrTy != PtrMemTy)
2624  return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2625  return SDValue(Node, 0);
2626 }
2627 
2628 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2629 /// tail spliced into a stack protector check success bb.
2630 ///
2631 /// For a high level explanation of how this fits into the stack protector
2632 /// generation see the comment on the declaration of class
2633 /// StackProtectorDescriptor.
2634 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2635  MachineBasicBlock *ParentBB) {
2636 
2637  // First create the loads to the guard/stack slot for the comparison.
2638  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2639  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2640  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2641 
2642  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2643  int FI = MFI.getStackProtectorIndex();
2644 
2645  SDValue Guard;
2646  SDLoc dl = getCurSDLoc();
2647  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2648  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2649  Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2650 
2651  // Generate code to load the content of the guard slot.
2652  SDValue GuardVal = DAG.getLoad(
2653  PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2656 
2657  if (TLI.useStackGuardXorFP())
2658  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2659 
2660  // Retrieve guard check function, nullptr if instrumentation is inlined.
2661  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2662  // The target provides a guard check function to validate the guard value.
2663  // Generate a call to that function with the content of the guard slot as
2664  // argument.
2665  FunctionType *FnTy = GuardCheckFn->getFunctionType();
2666  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2667 
2670  Entry.Node = GuardVal;
2671  Entry.Ty = FnTy->getParamType(0);
2672  if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2673  Entry.IsInReg = true;
2674  Args.push_back(Entry);
2675 
2677  CLI.setDebugLoc(getCurSDLoc())
2679  .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2680  getValue(GuardCheckFn), std::move(Args));
2681 
2682  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2683  DAG.setRoot(Result.second);
2684  return;
2685  }
2686 
2687  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2688  // Otherwise, emit a volatile load to retrieve the stack guard value.
2689  SDValue Chain = DAG.getEntryNode();
2690  if (TLI.useLoadStackGuardNode()) {
2691  Guard = getLoadStackGuard(DAG, dl, Chain);
2692  } else {
2693  const Value *IRGuard = TLI.getSDagStackGuard(M);
2694  SDValue GuardPtr = getValue(IRGuard);
2695 
2696  Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2697  MachinePointerInfo(IRGuard, 0), Align,
2699  }
2700 
2701  // Perform the comparison via a getsetcc.
2703  *DAG.getContext(),
2704  Guard.getValueType()),
2705  Guard, GuardVal, ISD::SETNE);
2706 
2707  // If the guard/stackslot do not equal, branch to failure MBB.
2708  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2709  MVT::Other, GuardVal.getOperand(0),
2710  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2711  // Otherwise branch to success MBB.
2712  SDValue Br = DAG.getNode(ISD::BR, dl,
2713  MVT::Other, BrCond,
2714  DAG.getBasicBlock(SPD.getSuccessMBB()));
2715 
2716  DAG.setRoot(Br);
2717 }
2718 
2719 /// Codegen the failure basic block for a stack protector check.
2720 ///
2721 /// A failure stack protector machine basic block consists simply of a call to
2722 /// __stack_chk_fail().
2723 ///
2724 /// For a high level explanation of how this fits into the stack protector
2725 /// generation see the comment on the declaration of class
2726 /// StackProtectorDescriptor.
2727 void
2728 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2729  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2731  CallOptions.setDiscardResult(true);
2732  SDValue Chain =
2733  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2734  None, CallOptions, getCurSDLoc()).second;
2735  // On PS4, the "return address" must still be within the calling function,
2736  // even if it's at the very end, so emit an explicit TRAP here.
2737  // Passing 'true' for doesNotReturn above won't generate the trap for us.
2738  if (TM.getTargetTriple().isPS4CPU())
2739  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2740  // WebAssembly needs an unreachable instruction after a non-returning call,
2741  // because the function return type can be different from __stack_chk_fail's
2742  // return type (void).
2743  if (TM.getTargetTriple().isWasm())
2744  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2745 
2746  DAG.setRoot(Chain);
2747 }
2748 
2749 /// visitBitTestHeader - This function emits necessary code to produce value
2750 /// suitable for "bit tests"
2752  MachineBasicBlock *SwitchBB) {
2753  SDLoc dl = getCurSDLoc();
2754 
2755  // Subtract the minimum value.
2756  SDValue SwitchOp = getValue(B.SValue);
2757  EVT VT = SwitchOp.getValueType();
2758  SDValue RangeSub =
2759  DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2760 
2761  // Determine the type of the test operands.
2762  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2763  bool UsePtrType = false;
2764  if (!TLI.isTypeLegal(VT)) {
2765  UsePtrType = true;
2766  } else {
2767  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2768  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2769  // Switch table case range are encoded into series of masks.
2770  // Just use pointer type, it's guaranteed to fit.
2771  UsePtrType = true;
2772  break;
2773  }
2774  }
2775  SDValue Sub = RangeSub;
2776  if (UsePtrType) {
2777  VT = TLI.getPointerTy(DAG.getDataLayout());
2778  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2779  }
2780 
2781  B.RegVT = VT.getSimpleVT();
2782  B.Reg = FuncInfo.CreateReg(B.RegVT);
2783  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2784 
2785  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2786 
2787  if (!B.FallthroughUnreachable)
2788  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2789  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2790  SwitchBB->normalizeSuccProbs();
2791 
2792  SDValue Root = CopyTo;
2793  if (!B.FallthroughUnreachable) {
2794  // Conditional branch to the default block.
2795  SDValue RangeCmp = DAG.getSetCC(dl,
2797  RangeSub.getValueType()),
2798  RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2799  ISD::SETUGT);
2800 
2801  Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2802  DAG.getBasicBlock(B.Default));
2803  }
2804 
2805  // Avoid emitting unnecessary branches to the next block.
2806  if (MBB != NextBlock(SwitchBB))
2807  Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2808 
2809  DAG.setRoot(Root);
2810 }
2811 
2812 /// visitBitTestCase - this function produces one "bit test"
2814  MachineBasicBlock* NextMBB,
2815  BranchProbability BranchProbToNext,
2816  unsigned Reg,
2817  BitTestCase &B,
2818  MachineBasicBlock *SwitchBB) {
2819  SDLoc dl = getCurSDLoc();
2820  MVT VT = BB.RegVT;
2821  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2822  SDValue Cmp;
2823  unsigned PopCount = countPopulation(B.Mask);
2824  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2825  if (PopCount == 1) {
2826  // Testing for a single bit; just compare the shift count with what it
2827  // would need to be to shift a 1 bit in that position.
2828  Cmp = DAG.getSetCC(
2829  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2830  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2831  ISD::SETEQ);
2832  } else if (PopCount == BB.Range) {
2833  // There is only one zero bit in the range, test for it directly.
2834  Cmp = DAG.getSetCC(
2835  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2836  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2837  ISD::SETNE);
2838  } else {
2839  // Make desired shift
2840  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2841  DAG.getConstant(1, dl, VT), ShiftOp);
2842 
2843  // Emit bit tests and jumps
2844  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2845  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2846  Cmp = DAG.getSetCC(
2847  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2848  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2849  }
2850 
2851  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2852  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2853  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2854  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2855  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2856  // one as they are relative probabilities (and thus work more like weights),
2857  // and hence we need to normalize them to let the sum of them become one.
2858  SwitchBB->normalizeSuccProbs();
2859 
2860  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2862  Cmp, DAG.getBasicBlock(B.TargetBB));
2863 
2864  // Avoid emitting unnecessary branches to the next block.
2865  if (NextMBB != NextBlock(SwitchBB))
2866  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2867  DAG.getBasicBlock(NextMBB));
2868 
2869  DAG.setRoot(BrAnd);
2870 }
2871 
2872 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2873  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2874 
2875  // Retrieve successors. Look through artificial IR level blocks like
2876  // catchswitch for successors.
2877  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2878  const BasicBlock *EHPadBB = I.getSuccessor(1);
2879 
2880  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2881  // have to do anything here to lower funclet bundles.
2882  assert(!I.hasOperandBundlesOtherThan(
2883  {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2884  LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2885  LLVMContext::OB_cfguardtarget,
2886  LLVMContext::OB_clang_arc_attachedcall}) &&
2887  "Cannot lower invokes with arbitrary operand bundles yet!");
2888 
2889  const Value *Callee(I.getCalledOperand());
2890  const Function *Fn = dyn_cast<Function>(Callee);
2891  if (isa<InlineAsm>(Callee))
2892  visitInlineAsm(I, EHPadBB);
2893  else if (Fn && Fn->isIntrinsic()) {
2894  switch (Fn->getIntrinsicID()) {
2895  default:
2896  llvm_unreachable("Cannot invoke this intrinsic");
2897  case Intrinsic::donothing:
2898  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2899  case Intrinsic::seh_try_begin:
2900  case Intrinsic::seh_scope_begin:
2901  case Intrinsic::seh_try_end:
2902  case Intrinsic::seh_scope_end:
2903  break;
2904  case Intrinsic::experimental_patchpoint_void:
2905  case Intrinsic::experimental_patchpoint_i64:
2906  visitPatchpoint(I, EHPadBB);
2907  break;
2908  case Intrinsic::experimental_gc_statepoint:
2909  LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2910  break;
2911  case Intrinsic::wasm_rethrow: {
2912  // This is usually done in visitTargetIntrinsic, but this intrinsic is
2913  // special because it can be invoked, so we manually lower it to a DAG
2914  // node here.
2916  Ops.push_back(getRoot()); // inchain
2917  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2918  Ops.push_back(
2919  DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2920  TLI.getPointerTy(DAG.getDataLayout())));
2921  SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2923  break;
2924  }
2925  }
2926  } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2927  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2928  // Eventually we will support lowering the @llvm.experimental.deoptimize
2929  // intrinsic, and right now there are no plans to support other intrinsics
2930  // with deopt state.
2931  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2932  } else {
2933  LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2934  }
2935 
2936  // If the value of the invoke is used outside of its defining block, make it
2937  // available as a virtual register.
2938  // We already took care of the exported value for the statepoint instruction
2939  // during call to the LowerStatepoint.
2940  if (!isa<GCStatepointInst>(I)) {
2942  }
2943 
2946  BranchProbability EHPadBBProb =
2947  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2949  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2950 
2951  // Update successor info.
2952  addSuccessorWithProb(InvokeMBB, Return);
2953  for (auto &UnwindDest : UnwindDests) {
2954  UnwindDest.first->setIsEHPad();
2955  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2956  }
2957  InvokeMBB->normalizeSuccProbs();
2958 
2959  // Drop into normal successor.
2961  DAG.getBasicBlock(Return)));
2962 }
2963 
2964 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2965  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2966 
2967  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2968  // have to do anything here to lower funclet bundles.
2969  assert(!I.hasOperandBundlesOtherThan(
2970  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2971  "Cannot lower callbrs with arbitrary operand bundles yet!");
2972 
2973  assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
2974  visitInlineAsm(I);
2976 
2977  // Retrieve successors.
2978  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2979 
2980  // Update successor info.
2981  addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
2982  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2983  MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2984  addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
2985  Target->setIsInlineAsmBrIndirectTarget();
2986  }
2987  CallBrMBB->normalizeSuccProbs();
2988 
2989  // Drop into default successor.
2992  DAG.getBasicBlock(Return)));
2993 }
2994 
2995 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2996  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2997 }
2998 
2999 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3000  assert(FuncInfo.MBB->isEHPad() &&
3001  "Call to landingpad not in landing pad!");
3002 
3003  // If there aren't registers to copy the values into (e.g., during SjLj
3004  // exceptions), then don't bother to create these DAG nodes.
3005  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3006  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3007  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3008  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3009  return;
3010 
3011  // If landingpad's return type is token type, we don't create DAG nodes
3012  // for its exception pointer and selector value. The extraction of exception
3013  // pointer or selector value from token type landingpads is not currently
3014  // supported.
3015  if (LP.getType()->isTokenTy())
3016  return;
3017 
3018  SmallVector<EVT, 2> ValueVTs;
3019  SDLoc dl = getCurSDLoc();
3020  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3021  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3022 
3023  // Get the two live-in registers as SDValues. The physregs have already been
3024  // copied into virtual registers.
3025  SDValue Ops[2];
3027  Ops[0] = DAG.getZExtOrTrunc(
3030  TLI.getPointerTy(DAG.getDataLayout())),
3031  dl, ValueVTs[0]);
3032  } else {
3033  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3034  }
3035  Ops[1] = DAG.getZExtOrTrunc(
3038  TLI.getPointerTy(DAG.getDataLayout())),
3039  dl, ValueVTs[1]);
3040 
3041  // Merge into one.
3043  DAG.getVTList(ValueVTs), Ops);
3044  setValue(&LP, Res);
3045 }
3046 
3048  MachineBasicBlock *Last) {
3049  // Update JTCases.
3050  for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
3051  if (SL->JTCases[i].first.HeaderBB == First)
3052  SL->JTCases[i].first.HeaderBB = Last;
3053 
3054  // Update BitTestCases.
3055  for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
3056  if (SL->BitTestCases[i].Parent == First)
3057  SL->BitTestCases[i].Parent = Last;
3058 }
3059 
3060 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3061  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3062 
3063  // Update machine-CFG edges with unique successors.
3065  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3066  BasicBlock *BB = I.getSuccessor(i);
3067  bool Inserted = Done.insert(BB).second;
3068  if (!Inserted)
3069  continue;
3070 
3072  addSuccessorWithProb(IndirectBrMBB, Succ);
3073  }
3074  IndirectBrMBB->normalizeSuccProbs();
3075 
3078  getValue(I.getAddress())));
3079 }
3080 
3081 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3083  return;
3084 
3085  // We may be able to ignore unreachable behind a noreturn call.
3087  const BasicBlock &BB = *I.getParent();
3088  if (&I != &BB.front()) {
3090  std::prev(BasicBlock::const_iterator(&I));
3091  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3092  if (Call->doesNotReturn())
3093  return;
3094  }
3095  }
3096  }
3097 
3099 }
3100 
3101 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3102  SDNodeFlags Flags;
3103 
3104  SDValue Op = getValue(I.getOperand(0));
3105  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3106  Op, Flags);
3107  setValue(&I, UnNodeValue);
3108 }
3109 
3110 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3111  SDNodeFlags Flags;
3112  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3113  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3114  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3115  }
3116  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3117  Flags.setExact(ExactOp->isExact());
3118  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3119  Flags.copyFMF(*FPOp);
3120 
3121  SDValue Op1 = getValue(I.getOperand(0));
3122  SDValue Op2 = getValue(I.getOperand(1));
3123  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3124  Op1, Op2, Flags);
3125  setValue(&I, BinNodeValue);
3126 }
3127 
3128 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3129  SDValue Op1 = getValue(I.getOperand(0));
3130  SDValue Op2 = getValue(I.getOperand(1));
3131 
3133  Op1.getValueType(), DAG.getDataLayout());
3134 
3135  // Coerce the shift amount to the right type if we can.
3136  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3137  unsigned ShiftSize = ShiftTy.getSizeInBits();
3138  unsigned Op2Size = Op2.getValueSizeInBits();
3139  SDLoc DL = getCurSDLoc();
3140 
3141  // If the operand is smaller than the shift count type, promote it.
3142  if (ShiftSize > Op2Size)
3143  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3144 
3145  // If the operand is larger than the shift count type but the shift
3146  // count type has enough bits to represent any shift value, truncate
3147  // it now. This is a common case and it exposes the truncate to
3148  // optimization early.
3149  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3150  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3151  // Otherwise we'll need to temporarily settle for some other convenient
3152  // type. Type legalization will make adjustments once the shiftee is split.
3153  else
3154  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3155  }
3156 
3157  bool nuw = false;
3158  bool nsw = false;
3159  bool exact = false;
3160 
3161  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3162 
3163  if (const OverflowingBinaryOperator *OFBinOp =
3164  dyn_cast<const OverflowingBinaryOperator>(&I)) {
3165  nuw = OFBinOp->hasNoUnsignedWrap();
3166  nsw = OFBinOp->hasNoSignedWrap();
3167  }
3168  if (const PossiblyExactOperator *ExactOp =
3169  dyn_cast<const PossiblyExactOperator>(&I))
3170  exact = ExactOp->isExact();
3171  }
3172  SDNodeFlags Flags;
3173  Flags.setExact(exact);
3174  Flags.setNoSignedWrap(nsw);
3175  Flags.setNoUnsignedWrap(nuw);
3176  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3177  Flags);
3178  setValue(&I, Res);
3179 }
3180 
3181 void SelectionDAGBuilder::visitSDiv(const User &I) {
3182  SDValue Op1 = getValue(I.getOperand(0));
3183  SDValue Op2 = getValue(I.getOperand(1));
3184 
3185  SDNodeFlags Flags;
3186  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3187  cast<PossiblyExactOperator>(&I)->isExact());
3189  Op2, Flags));
3190 }
3191 
3192 void SelectionDAGBuilder::visitICmp(const User &I) {
3194  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3195  predicate = IC->getPredicate();
3196  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3197  predicate = ICmpInst::Predicate(IC->getPredicate());
3198  SDValue Op1 = getValue(I.getOperand(0));
3199  SDValue Op2 = getValue(I.getOperand(1));
3200  ISD::CondCode Opcode = getICmpCondCode(predicate);
3201 
3202  auto &TLI = DAG.getTargetLoweringInfo();
3203  EVT MemVT =
3204  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3205 
3206  // If a pointer's DAG type is larger than its memory type then the DAG values
3207  // are zero-extended. This breaks signed comparisons so truncate back to the
3208  // underlying type before doing the compare.
3209  if (Op1.getValueType() != MemVT) {
3210  Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3211  Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3212  }
3213 
3215  I.getType());
3216  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3217 }
3218 
3219 void SelectionDAGBuilder::visitFCmp(const User &I) {
3221  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3222  predicate = FC->getPredicate();
3223  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3224  predicate = FCmpInst::Predicate(FC->getPredicate());
3225  SDValue Op1 = getValue(I.getOperand(0));
3226  SDValue Op2 = getValue(I.getOperand(1));
3227 
3228  ISD::CondCode Condition = getFCmpCondCode(predicate);
3229  auto *FPMO = cast<FPMathOperator>(&I);
3230  if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3231  Condition = getFCmpCodeWithoutNaN(Condition);
3232 
3233  SDNodeFlags Flags;
3234  Flags.copyFMF(*FPMO);
3235  SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3236 
3238  I.getType());
3239  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3240 }
3241 
3242 // Check if the condition of the select has one use or two users that are both
3243 // selects with the same condition.
3244 static bool hasOnlySelectUsers(const Value *Cond) {
3245  return llvm::all_of(Cond->users(), [](const Value *V) {
3246  return isa<SelectInst>(V);
3247  });
3248 }
3249 
3250 void SelectionDAGBuilder::visitSelect(const User &I) {
3251  SmallVector<EVT, 4> ValueVTs;
3253  ValueVTs);
3254  unsigned NumValues = ValueVTs.size();
3255  if (NumValues == 0) return;
3256 
3257  SmallVector<SDValue, 4> Values(NumValues);
3258  SDValue Cond = getValue(I.getOperand(0));
3259  SDValue LHSVal = getValue(I.getOperand(1));
3260  SDValue RHSVal = getValue(I.getOperand(2));
3261  SmallVector<SDValue, 1> BaseOps(1, Cond);
3262  ISD::NodeType OpCode =
3263  Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3264 
3265  bool IsUnaryAbs = false;
3266  bool Negate = false;
3267 
3268  SDNodeFlags Flags;
3269  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3270  Flags.copyFMF(*FPOp);
3271 
3272  // Min/max matching is only viable if all output VTs are the same.
3273  if (is_splat(ValueVTs)) {
3274  EVT VT = ValueVTs[0];
3275  LLVMContext &Ctx = *DAG.getContext();
3276  auto &TLI = DAG.getTargetLoweringInfo();
3277 
3278  // We care about the legality of the operation after it has been type
3279  // legalized.
3280  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3281  VT = TLI.getTypeToTransformTo(Ctx, VT);
3282 
3283  // If the vselect is legal, assume we want to leave this as a vector setcc +
3284  // vselect. Otherwise, if this is going to be scalarized, we want to see if
3285  // min/max is legal on the scalar type.
3286  bool UseScalarMinMax = VT.isVector() &&
3288 
3289  Value *LHS, *RHS;
3290  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3292  switch (SPR.Flavor) {
3293  case SPF_UMAX: Opc = ISD::UMAX; break;
3294  case SPF_UMIN: Opc = ISD::UMIN; break;
3295  case SPF_SMAX: Opc = ISD::SMAX; break;
3296  case SPF_SMIN: Opc = ISD::SMIN; break;
3297  case SPF_FMINNUM:
3298  switch (SPR.NaNBehavior) {
3299  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3300  case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3301  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3302  case SPNB_RETURNS_ANY: {
3304  Opc = ISD::FMINNUM;
3305  else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3306  Opc = ISD::FMINIMUM;
3307  else if (UseScalarMinMax)
3310  break;
3311  }
3312  }
3313  break;
3314  case SPF_FMAXNUM:
3315  switch (SPR.NaNBehavior) {
3316  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3317  case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3318  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3319  case SPNB_RETURNS_ANY:
3320 
3322  Opc = ISD::FMAXNUM;
3323  else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3324  Opc = ISD::FMAXIMUM;
3325  else if (UseScalarMinMax)
3328  break;
3329  }
3330  break;
3331  case SPF_NABS:
3332  Negate = true;
3334  case SPF_ABS:
3335  IsUnaryAbs = true;
3336  Opc = ISD::ABS;
3337  break;
3338  default: break;
3339  }
3340 
3341  if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3342  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3343  (UseScalarMinMax &&
3344  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3345  // If the underlying comparison instruction is used by any other
3346  // instruction, the consumed instructions won't be destroyed, so it is
3347  // not profitable to convert to a min/max.
3348  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3349  OpCode = Opc;
3350  LHSVal = getValue(LHS);
3351  RHSVal = getValue(RHS);
3352  BaseOps.clear();
3353  }
3354 
3355  if (IsUnaryAbs) {
3356  OpCode = Opc;
3357  LHSVal = getValue(LHS);
3358  BaseOps.clear();
3359  }
3360  }
3361 
3362  if (IsUnaryAbs) {
3363  for (unsigned i = 0; i != NumValues; ++i) {
3364  SDLoc dl = getCurSDLoc();
3365  EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3366  Values[i] =
3367  DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3368  if (Negate)
3369  Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
3370  Values[i]);
3371  }
3372  } else {
3373  for (unsigned i = 0; i != NumValues; ++i) {
3374  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3375  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3376  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3377  Values[i] = DAG.getNode(
3378  OpCode, getCurSDLoc(),
3379  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3380  }
3381  }
3382 
3384  DAG.getVTList(ValueVTs), Values));
3385 }
3386 
3387 void SelectionDAGBuilder::visitTrunc(const User &I) {
3388  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3389  SDValue N = getValue(I.getOperand(0));
3391  I.getType());
3392  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3393 }
3394 
3395 void SelectionDAGBuilder::visitZExt(const User &I) {
3396  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3397  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3398  SDValue N = getValue(I.getOperand(0));
3400  I.getType());
3402 }
3403 
3404 void SelectionDAGBuilder::visitSExt(const User &I) {
3405  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3406  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3407  SDValue N = getValue(I.getOperand(0));
3409  I.getType());
3411 }
3412 
3413 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3414  // FPTrunc is never a no-op cast, no need to check
3415  SDValue N = getValue(I.getOperand(0));
3416  SDLoc dl = getCurSDLoc();
3417  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3418  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3419  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3421  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3422 }
3423 
3424 void SelectionDAGBuilder::visitFPExt(const User &I) {
3425  // FPExt is never a no-op cast, no need to check
3426  SDValue N = getValue(I.getOperand(0));
3428  I.getType());
3429  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3430 }
3431 
3432 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3433  // FPToUI is never a no-op cast, no need to check
3434  SDValue N = getValue(I.getOperand(0));
3436  I.getType());
3437  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3438 }
3439 
3440 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3441  // FPToSI is never a no-op cast, no need to check
3442  SDValue N = getValue(I.getOperand(0));
3444  I.getType());
3445  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3446 }
3447 
3448 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3449  // UIToFP is never a no-op cast, no need to check
3450  SDValue N = getValue(I.getOperand(0));
3452  I.getType());
3453  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3454 }
3455 
3456 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3457  // SIToFP is never a no-op cast, no need to check
3458  SDValue N = getValue(I.getOperand(0));
3460  I.getType());
3461  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3462 }
3463 
3464 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3465  // What to do depends on the size of the integer and the size of the pointer.
3466  // We can either truncate, zero extend, or no-op, accordingly.
3467  SDValue N = getValue(I.getOperand(0));
3468  auto &TLI = DAG.getTargetLoweringInfo();
3470  I.getType());
3471  EVT PtrMemVT =
3472  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3473  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3474  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3475  setValue(&I, N);
3476 }
3477 
3478 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3479  // What to do depends on the size of the integer and the size of the pointer.
3480  // We can either truncate, zero extend, or no-op, accordingly.
3481  SDValue N = getValue(I.getOperand(0));
3482  auto &TLI = DAG.getTargetLoweringInfo();
3483  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3484  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3485  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3486  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3487  setValue(&I, N);
3488 }
3489 
3490 void SelectionDAGBuilder::visitBitCast(const User &I) {
3491  SDValue N = getValue(I.getOperand(0));
3492  SDLoc dl = getCurSDLoc();
3494  I.getType());
3495 
3496  // BitCast assures us that source and destination are the same size so this is
3497  // either a BITCAST or a no-op.
3498  if (DestVT != N.getValueType())
3500  DestVT, N)); // convert types.
3501  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3502  // might fold any kind of constant expression to an integer constant and that
3503  // is not what we are looking for. Only recognize a bitcast of a genuine
3504  // constant integer as an opaque constant.
3505  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3506  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3507  /*isOpaque*/true));
3508  else
3509  setValue(&I, N); // noop cast.
3510 }
3511 
3512 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3513  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3514  const Value *SV = I.getOperand(0);
3515  SDValue N = getValue(SV);
3516  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3517 
3518  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3519  unsigned DestAS = I.getType()->getPointerAddressSpace();
3520 
3521  if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3522  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3523 
3524  setValue(&I, N);
3525 }
3526 
3527 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3528  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3529  SDValue InVec = getValue(I.getOperand(0));
3530  SDValue InVal = getValue(I.getOperand(1));
3531  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3534  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3535  InVec, InVal, InIdx));
3536 }
3537 
3538 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3539  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3540  SDValue InVec = getValue(I.getOperand(0));
3541  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3544  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3545  InVec, InIdx));
3546 }
3547 
3548 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3549  SDValue Src1 = getValue(I.getOperand(0));
3550  SDValue Src2 = getValue(I.getOperand(1));
3552  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3553  Mask = SVI->getShuffleMask();
3554  else
3555  Mask = cast<ConstantExpr>(I).getShuffleMask();
3556  SDLoc DL = getCurSDLoc();
3557  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3558  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3559  EVT SrcVT = Src1.getValueType();
3560 
3561  if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3562  VT.isScalableVector()) {
3563  // Canonical splat form of first element of first input vector.
3564  SDValue FirstElt =
3567  setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3568  return;
3569  }
3570 
3571  // For now, we only handle splats for scalable vectors.
3572  // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3573  // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3574  assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3575 
3576  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3577  unsigned MaskNumElts = Mask.size();
3578 
3579  if (SrcNumElts == MaskNumElts) {
3580  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3581  return;
3582  }
3583 
3584  // Normalize the shuffle vector since mask and vector length don't match.
3585  if (SrcNumElts < MaskNumElts) {
3586  // Mask is longer than the source vectors. We can use concatenate vector to
3587  // make the mask and vectors lengths match.
3588 
3589  if (MaskNumElts % SrcNumElts == 0) {
3590  // Mask length is a multiple of the source vector length.
3591  // Check if the shuffle is some kind of concatenation of the input
3592  // vectors.
3593  unsigned NumConcat = MaskNumElts / SrcNumElts;
3594  bool IsConcat = true;
3595  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3596  for (unsigned i = 0; i != MaskNumElts; ++i) {
3597  int Idx = Mask[i];
3598  if (Idx < 0)
3599  continue;
3600  // Ensure the indices in each SrcVT sized piece are sequential and that
3601  // the same source is used for the whole piece.
3602  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3603  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3604  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3605  IsConcat = false;
3606  break;
3607  }
3608  // Remember which source this index came from.
3609  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3610  }
3611 
3612  // The shuffle is concatenating multiple vectors together. Just emit
3613  // a CONCAT_VECTORS operation.
3614  if (IsConcat) {
3615  SmallVector<SDValue, 8> ConcatOps;
3616  for (auto Src : ConcatSrcs) {
3617  if (Src < 0)
3618  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3619  else if (Src == 0)
3620  ConcatOps.push_back(Src1);
3621  else
3622  ConcatOps.push_back(Src2);
3623  }
3624  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3625  return;
3626  }
3627  }
3628 
3629  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3630  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3631  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3632  PaddedMaskNumElts);
3633 
3634  // Pad both vectors with undefs to make them the same length as the mask.
3635  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3636 
3637  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3638  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3639  MOps1[0] = Src1;
3640  MOps2[0] = Src2;
3641 
3642  Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3643  Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3644 
3645  // Readjust mask for new input vector length.
3646  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3647  for (unsigned i = 0; i != MaskNumElts; ++i) {
3648  int Idx = Mask[i];
3649  if (Idx >= (int)SrcNumElts)
3650  Idx -= SrcNumElts - PaddedMaskNumElts;
3651  MappedOps[i] = Idx;
3652  }
3653 
3654  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3655 
3656  // If the concatenated vector was padded, extract a subvector with the
3657  // correct number of elements.
3658  if (MaskNumElts != PaddedMaskNumElts)
3659  Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3661 
3662  setValue(&I, Result);
3663  return;
3664  }
3665 
3666  if (SrcNumElts > MaskNumElts) {
3667  // Analyze the access pattern of the vector to see if we can extract
3668  // two subvectors and do the shuffle.
3669  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3670  bool CanExtract = true;
3671  for (int Idx : Mask) {
3672  unsigned Input = 0;
3673  if (Idx < 0)
3674  continue;
3675 
3676  if (Idx >= (int)SrcNumElts) {
3677  Input = 1;
3678  Idx -= SrcNumElts;
3679  }
3680 
3681  // If all the indices come from the same MaskNumElts sized portion of
3682  // the sources we can use extract. Also make sure the extract wouldn't
3683  // extract past the end of the source.
3684  int NewStartIdx = alignDown(Idx, MaskNumElts);
3685  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3686  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3687  CanExtract = false;
3688  // Make sure we always update StartIdx as we use it to track if all
3689  // elements are undef.
3690  StartIdx[Input] = NewStartIdx;
3691  }
3692 
3693  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3694  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3695  return;
3696  }
3697  if (CanExtract) {
3698  // Extract appropriate subvector and generate a vector shuffle
3699  for (unsigned Input = 0; Input < 2; ++Input) {
3700  SDValue &Src = Input == 0 ? Src1 : Src2;
3701  if (StartIdx[Input] < 0)
3702  Src = DAG.getUNDEF(VT);
3703  else {
3704  Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3705  DAG.getVectorIdxConstant(StartIdx[Input], DL));
3706  }
3707  }
3708 
3709  // Calculate new mask.
3710  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3711  for (int &Idx : MappedOps) {
3712  if (Idx >= (int)SrcNumElts)
3713  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3714  else if (Idx >= 0)
3715  Idx -= StartIdx[0];
3716  }
3717 
3718  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3719  return;
3720  }
3721  }
3722 
3723  // We can't use either concat vectors or extract subvectors so fall back to
3724  // replacing the shuffle with extract and build vector.
3725  // to insert and build vector.
3726  EVT EltVT = VT.getVectorElementType();
3728  for (int Idx : Mask) {
3729  SDValue Res;
3730 
3731  if (Idx < 0) {
3732  Res = DAG.getUNDEF(EltVT);
3733  } else {
3734  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3735  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3736 
3737  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3738  DAG.getVectorIdxConstant(Idx, DL));
3739  }
3740 
3741  Ops.push_back(Res);
3742  }
3743 
3744  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3745 }
3746 
3747 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3748  ArrayRef<unsigned> Indices;
3749  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3750  Indices = IV->getIndices();
3751  else
3752  Indices = cast<ConstantExpr>(&I)->getIndices();
3753 
3754  const Value *Op0 = I.getOperand(0);
3755  const Value *Op1 = I.getOperand(1);
3756  Type *AggTy = I.getType();
3757  Type *ValTy = Op1->getType();
3758  bool IntoUndef = isa<UndefValue>(Op0);
3759  bool FromUndef = isa<UndefValue>(Op1);
3760 
3761  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3762 
3763  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3764  SmallVector<EVT, 4> AggValueVTs;
3765  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3766  SmallVector<EVT, 4> ValValueVTs;
3767  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3768 
3769  unsigned NumAggValues = AggValueVTs.size();
3770  unsigned NumValValues = ValValueVTs.size();
3771  SmallVector<SDValue, 4> Values(NumAggValues);
3772 
3773  // Ignore an insertvalue that produces an empty object
3774  if (!NumAggValues) {
3776  return;
3777  }
3778 
3779  SDValue Agg = getValue(Op0);
3780  unsigned i = 0;
3781  // Copy the beginning value(s) from the original aggregate.
3782  for (; i != LinearIndex; ++i)
3783  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3784  SDValue(Agg.getNode(), Agg.getResNo() + i);
3785  // Copy values from the inserted value(s).
3786  if (NumValValues) {
3787  SDValue Val = getValue(Op1);
3788  for (; i != LinearIndex + NumValValues; ++i)
3789  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3790  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3791  }
3792  // Copy remaining value(s) from the original aggregate.
3793  for (; i != NumAggValues; ++i)
3794  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3795  SDValue(Agg.getNode(), Agg.getResNo() + i);
3796 
3798  DAG.getVTList(AggValueVTs), Values));
3799 }
3800 
3801 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3802  ArrayRef<unsigned> Indices;
3803  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3804  Indices = EV->getIndices();
3805  else
3806  Indices = cast<ConstantExpr>(&I)->getIndices();
3807 
3808  const Value *Op0 = I.getOperand(0);
3809  Type *AggTy = Op0->getType();
3810  Type *ValTy = I.getType();
3811  bool OutOfUndef = isa<UndefValue>(Op0);
3812 
3813  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3814 
3815  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3816  SmallVector<EVT, 4> ValValueVTs;
3817  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3818 
3819  unsigned NumValValues = ValValueVTs.size();
3820 
3821  // Ignore a extractvalue that produces an empty object
3822  if (!NumValValues) {
3824  return;
3825  }
3826 
3827  SmallVector<SDValue, 4> Values(NumValValues);
3828 
3829  SDValue Agg = getValue(Op0);
3830  // Copy out the selected value(s).
3831  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3832  Values[i - LinearIndex] =
3833  OutOfUndef ?
3834  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3835  SDValue(Agg.getNode(), Agg.getResNo() + i);
3836 
3838  DAG.getVTList(ValValueVTs), Values));
3839 }
3840 
3841 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3842  Value *Op0 = I.getOperand(0);
3843  // Note that the pointer operand may be a vector of pointers. Take the scalar
3844  // element which holds a pointer.
3845  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3846  SDValue N = getValue(Op0);
3847  SDLoc dl = getCurSDLoc();
3848  auto &TLI = DAG.getTargetLoweringInfo();
3849 
3850  // Normalize Vector GEP - all scalar operands should be converted to the
3851  // splat vector.
3852  bool IsVectorGEP = I.getType()->isVectorTy();
3853  ElementCount VectorElementCount =
3854  IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3856 
3857  if (IsVectorGEP && !N.getValueType().isVector()) {
3859  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3860  if (VectorElementCount.isScalable())
3861  N = DAG.getSplatVector(VT, dl, N);
3862  else
3863  N = DAG.getSplatBuildVector(VT, dl, N);
3864  }
3865 
3866  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3867  GTI != E; ++GTI) {
3868  const Value *Idx = GTI.getOperand();
3869  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3870  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3871  if (Field) {
3872  // N = N + Offset
3874 
3875  // In an inbounds GEP with an offset that is nonnegative even when
3876  // interpreted as signed, assume there is no unsigned overflow.
3877  SDNodeFlags Flags;
3878  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3879  Flags.setNoUnsignedWrap(true);
3880 
3881  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3882  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3883  }
3884  } else {
3885  // IdxSize is the width of the arithmetic according to IR semantics.
3886  // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3887  // (and fix up the result later).
3888  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3889  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3890  TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
3891  // We intentionally mask away the high bits here; ElementSize may not
3892  // fit in IdxTy.
3893  APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3894  bool ElementScalable = ElementSize.isScalable();
3895 
3896  // If this is a scalar constant or a splat vector of constants,
3897  // handle it quickly.
3898  const auto *C = dyn_cast<Constant>(Idx);
3899  if (C && isa<VectorType>(C->getType()))
3900  C = C->getSplatValue();
3901 
3902  const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3903  if (CI && CI->isZero())
3904  continue;
3905  if (CI && !ElementScalable) {
3906  APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3908  SDValue OffsVal;
3909  if (IsVectorGEP)
3910  OffsVal = DAG.getConstant(
3911  Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3912  else
3913  OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3914 
3915  // In an inbounds GEP with an offset that is nonnegative even when
3916  // interpreted as signed, assume there is no unsigned overflow.
3917  SDNodeFlags Flags;
3918  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3919  Flags.setNoUnsignedWrap(true);
3920 
3921  OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3922 
3923  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3924  continue;
3925  }
3926 
3927  // N = N + Idx * ElementMul;
3928  SDValue IdxN = getValue(Idx);
3929 
3930  if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3931  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3932  VectorElementCount);
3933  if (VectorElementCount.isScalable())
3934  IdxN = DAG.getSplatVector(VT, dl, IdxN);
3935  else
3936  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3937  }
3938 
3939  // If the index is smaller or larger than intptr_t, truncate or extend
3940  // it.
3941  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3942 
3943  if (ElementScalable) {
3944  EVT VScaleTy = N.getValueType().getScalarType();
3945  SDValue VScale = DAG.getNode(
3946  ISD::VSCALE, dl, VScaleTy,
3947  DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3948  if (IsVectorGEP)
3949  VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3950  IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3951  } else {
3952  // If this is a multiply by a power of two, turn it into a shl
3953  // immediately. This is a very common case.
3954  if (ElementMul != 1) {
3955  if (ElementMul.isPowerOf2()) {
3956  unsigned Amt = ElementMul.logBase2();
3957  IdxN = DAG.getNode(ISD::SHL, dl,
3958  N.getValueType(), IdxN,
3959  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3960  } else {
3961  SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3962  IdxN.getValueType());
3963  IdxN = DAG.getNode(ISD::MUL, dl,
3964  N.getValueType(), IdxN, Scale);
3965  }
3966  }
3967  }
3968 
3969  N = DAG.getNode(ISD::ADD, dl,
3970  N.getValueType(), N, IdxN);
3971  }
3972  }
3973 
3974  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3975  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3976  if (IsVectorGEP) {
3977  PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
3978  PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
3979  }
3980 
3981  if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3982  N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3983 
3984  setValue(&I, N);
3985 }
3986 
3987 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3988  // If this is a fixed sized alloca in the entry block of the function,
3989  // allocate it statically on the stack.
3990  if (FuncInfo.StaticAllocaMap.count(&I))
3991  return; // getValue will auto-populate this.
3992 
3993  SDLoc dl = getCurSDLoc();
3994  Type *Ty = I.getAllocatedType();
3995  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3996  auto &DL = DAG.getDataLayout();
3997  uint64_t TySize = DL.getTypeAllocSize(Ty);
3998  MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
3999 
4000  SDValue AllocSize = getValue(I.getArraySize());
4001 
4003  if (AllocSize.getValueType() != IntPtr)
4004  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4005 
4006  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
4007  AllocSize,
4008  DAG.getConstant(TySize, dl, IntPtr));
4009 
4010  // Handle alignment. If the requested alignment is less than or equal to
4011  // the stack alignment, ignore it. If the size is greater than or equal to
4012  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4014  if (*Alignment <= StackAlign)
4015  Alignment = None;
4016 
4017  const uint64_t StackAlignMask = StackAlign.value() - 1U;
4018  // Round the size of the allocation up to the stack alignment size
4019  // by add SA-1 to the size. This doesn't overflow because we're computing
4020  // an address inside an alloca.
4021  SDNodeFlags Flags;
4022  Flags.setNoUnsignedWrap(true);
4023  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4024  DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4025 
4026  // Mask out the low bits for alignment purposes.
4027  AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4028  DAG.getConstant(~StackAlignMask, dl, IntPtr));
4029 
4030  SDValue Ops[] = {
4031  getRoot(), AllocSize,
4032  DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4033  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4034  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4035  setValue(&I, DSA);
4036  DAG.setRoot(DSA.getValue(1));
4037 
4039 }
4040 
4041 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4042  if (I.isAtomic())
4043  return visitAtomicLoad(I);
4044 
4045  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4046  const Value *SV = I.getOperand(0);
4047  if (TLI.supportSwiftError()) {
4048  // Swifterror values can come from either a function parameter with
4049  // swifterror attribute or an alloca with swifterror attribute.
4050  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4051  if (Arg->hasSwiftErrorAttr())
4052  return visitLoadFromSwiftError(I);
4053  }
4054 
4055  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4056  if (Alloca->isSwiftError())
4057  return visitLoadFromSwiftError(I);
4058  }
4059  }
4060 
4061  SDValue Ptr = getValue(SV);
4062 
4063  Type *Ty = I.getType();
4064  Align Alignment = I.getAlign();
4065 
4066  AAMDNodes AAInfo = I.getAAMetadata();
4067  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4068 
4069  SmallVector<EVT, 4> ValueVTs, MemVTs;
4071  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4072  unsigned NumValues = ValueVTs.size();
4073  if (NumValues == 0)
4074  return;
4075 
4076  bool isVolatile = I.isVolatile();
4077 
4078  SDValue Root;
4079  bool ConstantMemory = false;
4080  if (isVolatile)
4081  // Serialize volatile loads with other side effects.
4082  Root = getRoot();
4083  else if (NumValues > MaxParallelChains)
4084  Root = getMemoryRoot();
4085  else if (AA &&
4087  SV,
4089  AAInfo))) {
4090  // Do not serialize (non-volatile) loads of constant memory with anything.
4091  Root = DAG.getEntryNode();
4092  ConstantMemory = true;
4093  } else {
4094  // Do not serialize non-volatile loads against each other.
4095  Root = DAG.getRoot();
4096  }
4097 
4098  SDLoc dl = getCurSDLoc();
4099 
4100  if (isVolatile)
4101  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4102 
4103  // An aggregate load cannot wrap around the address space, so offsets to its
4104  // parts don't wrap either.
4105  SDNodeFlags Flags;
4106  Flags.setNoUnsignedWrap(true);
4107 
4108  SmallVector<SDValue, 4> Values(NumValues);
4110  EVT PtrVT = Ptr.getValueType();
4111 
4112  MachineMemOperand::Flags MMOFlags
4114 
4115  unsigned ChainI = 0;
4116  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4117  // Serializing loads here may result in excessive register pressure, and
4118  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4119  // could recover a bit by hoisting nodes upward in the chain by recognizing
4120  // they are side-effect free or do not alias. The optimizer should really
4121  // avoid this case by converting large object/array copies to llvm.memcpy
4122  // (MaxParallelChains should always remain as failsafe).
4123  if (ChainI == MaxParallelChains) {
4124  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4126  makeArrayRef(Chains.data(), ChainI));
4127  Root = Chain;
4128  ChainI = 0;
4129  }
4130  SDValue A = DAG.getNode(ISD::ADD, dl,
4131  PtrVT, Ptr,
4132  DAG.getConstant(Offsets[i], dl, PtrVT),
4133  Flags);
4134 
4135  SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4136  MachinePointerInfo(SV, Offsets[i]), Alignment,
4137  MMOFlags, AAInfo, Ranges);
4138  Chains[ChainI] = L.getValue(1);
4139 
4140  if (MemVTs[i] != ValueVTs[i])
4141  L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4142 
4143  Values[i] = L;
4144  }
4145 
4146  if (!ConstantMemory) {
4148  makeArrayRef(Chains.data(), ChainI));
4149  if (isVolatile)
4150  DAG.setRoot(Chain);
4151  else
4152  PendingLoads.push_back(Chain);
4153  }
4154 
4156  DAG.getVTList(ValueVTs), Values));
4157 }
4158 
4159 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4161  "call visitStoreToSwiftError when backend supports swifterror");
4162 
4163  SmallVector<EVT, 4> ValueVTs;
4165  const Value *SrcV = I.getOperand(0);
4167  SrcV->getType(), ValueVTs, &Offsets);
4168  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4169  "expect a single EVT for swifterror");
4170 
4171  SDValue Src = getValue(SrcV);
4172  // Create a virtual register, then update the virtual register.
4173  Register VReg =
4174  SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4175  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4176  // Chain can be getRoot or getControlRoot.
4177  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4178  SDValue(Src.getNode(), Src.getResNo()));
4179  DAG.setRoot(CopyNode);
4180 }
4181 
4182 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4184  "call visitLoadFromSwiftError when backend supports swifterror");
4185 
4186  assert(!I.isVolatile() &&
4187  !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4188  !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4189  "Support volatile, non temporal, invariant for load_from_swift_error");
4190 
4191  const Value *SV = I.getOperand(0);
4192  Type *Ty = I.getType();
4193  assert(
4194  (!AA ||
4197  I.getAAMetadata()))) &&
4198  "load_from_swift_error should not be constant memory");
4199 
4200  SmallVector<EVT, 4> ValueVTs;
4203  ValueVTs, &Offsets);
4204  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4205  "expect a single EVT for swifterror");
4206 
4207  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4209  getRoot(), getCurSDLoc(),
4210  SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4211 
4212  setValue(&I, L);
4213 }
4214 
4215 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4216  if (I.isAtomic())
4217  return visitAtomicStore(I);
4218 
4219  const Value *SrcV = I.getOperand(0);
4220  const Value *PtrV = I.getOperand(1);
4221 
4222  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4223  if (TLI.supportSwiftError()) {
4224  // Swifterror values can come from either a function parameter with
4225  // swifterror attribute or an alloca with swifterror attribute.
4226  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4227  if (Arg->hasSwiftErrorAttr())
4228  return visitStoreToSwiftError(I);
4229  }
4230 
4231  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4232  if (Alloca->isSwiftError())
4233  return visitStoreToSwiftError(I);
4234  }
4235  }
4236 
4237  SmallVector<EVT, 4> ValueVTs, MemVTs;
4240  SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4241  unsigned NumValues = ValueVTs.size();
4242  if (NumValues == 0)
4243  return;
4244 
4245  // Get the lowered operands. Note that we do this after
4246  // checking if NumResults is zero, because with zero results
4247  // the operands won't have values in the map.
4248  SDValue Src = getValue(SrcV);
4249  SDValue Ptr = getValue(PtrV);
4250 
4251  SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4253  SDLoc dl = getCurSDLoc();
4254  Align Alignment = I.getAlign();
4255  AAMDNodes AAInfo = I.getAAMetadata();
4256 
4257  auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4258 
4259  // An aggregate load cannot wrap around the address space, so offsets to its
4260  // parts don't wrap either.
4261  SDNodeFlags Flags;
4262  Flags.setNoUnsignedWrap(true);
4263 
4264  unsigned ChainI = 0;
4265  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4266  // See visitLoad comments.
4267  if (ChainI == MaxParallelChains) {
4269  makeArrayRef(Chains.data(), ChainI));
4270  Root = Chain;
4271  ChainI = 0;
4272  }
4273  SDValue Add =
4274  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4275  SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4276  if (MemVTs[i] != ValueVTs[i])
4277  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4278  SDValue St =
4279  DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4280  Alignment, MMOFlags, AAInfo);
4281  Chains[ChainI] = St;
4282  }
4283 
4284  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4285  makeArrayRef(Chains.data(), ChainI));
4286  DAG.setRoot(StoreNode);
4287 }
4288 
4289 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4290  bool IsCompressing) {
4291  SDLoc sdl = getCurSDLoc();
4292 
4293  auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4294  MaybeAlign &Alignment) {
4295  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4296  Src0 = I.getArgOperand(0);
4297  Ptr = I.getArgOperand(1);
4298  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4299  Mask = I.getArgOperand(3);
4300  };
4301  auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4302  MaybeAlign &Alignment) {
4303  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4304  Src0 = I.getArgOperand(0);
4305  Ptr = I.getArgOperand(1);
4306  Mask = I.getArgOperand(2);
4307  Alignment = None;
4308  };
4309 
4310  Value *PtrOperand, *MaskOperand, *Src0Operand;
4311  MaybeAlign Alignment;
4312  if (IsCompressing)
4313  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4314  else
4315  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4316 
4317  SDValue Ptr = getValue(PtrOperand);
4318  SDValue Src0 = getValue(Src0Operand);
4319  SDValue Mask = getValue(MaskOperand);
4321 
4322  EVT VT = Src0.getValueType();
4323  if (!Alignment)
4324  Alignment = DAG.getEVTAlign(VT);
4325 
4328  // TODO: Make MachineMemOperands aware of scalable
4329  // vectors.
4330  VT.getStoreSize().getKnownMinSize(), *Alignment, I.getAAMetadata());
4331  SDValue StoreNode =
4332  DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4333  ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4334  DAG.setRoot(StoreNode);
4335  setValue(&I, StoreNode);
4336 }
4337 
4338 // Get a uniform base for the Gather/Scatter intrinsic.
4339 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4340 // We try to represent it as a base pointer + vector of indices.
4341 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4342 // The first operand of the GEP may be a single pointer or a vector of pointers
4343 // Example:
4344 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4345 // or
4346 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4347 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4348 //
4349 // When the first GEP operand is a single pointer - it is the uniform base we
4350 // are looking for. If first operand of the GEP is a splat vector - we
4351 // extract the splat value and use it as a uniform base.
4352 // In all other cases the function returns 'false'.
4353 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4354  ISD::MemIndexType &IndexType, SDValue &Scale,
4355  SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {
4356  SelectionDAG& DAG = SDB->DAG;
4357  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4358  const DataLayout &DL = DAG.getDataLayout();
4359 
4360  assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4361 
4362  // Handle splat constant pointer.
4363  if (auto *C = dyn_cast<Constant>(Ptr)) {
4364  C = C->getSplatValue();
4365  if (!C)
4366  return false;
4367 
4368  Base = SDB->getValue(C);
4369 
4370  ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4371  EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4372  Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4373  IndexType = ISD::SIGNED_SCALED;
4374  Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4375  return true;
4376  }
4377 
4378  const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4379  if (!GEP || GEP->getParent() != CurBB)
4380  return false;
4381 
4382  if (GEP->getNumOperands() != 2)
4383  return false;
4384 
4385  const Value *BasePtr = GEP->getPointerOperand();
4386  const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4387 
4388  // Make sure the base is scalar and the index is a vector.
4389  if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4390  return false;
4391 
4392