LLVM  16.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/ADT/Twine.h"
29 #include "llvm/Analysis/Loads.h"
33 #include "llvm/CodeGen/Analysis.h"
49 #include "llvm/CodeGen/StackMaps.h"
57 #include "llvm/IR/Argument.h"
58 #include "llvm/IR/Attributes.h"
59 #include "llvm/IR/BasicBlock.h"
60 #include "llvm/IR/CFG.h"
61 #include "llvm/IR/CallingConv.h"
62 #include "llvm/IR/Constant.h"
63 #include "llvm/IR/ConstantRange.h"
64 #include "llvm/IR/Constants.h"
65 #include "llvm/IR/DataLayout.h"
67 #include "llvm/IR/DerivedTypes.h"
68 #include "llvm/IR/DiagnosticInfo.h"
69 #include "llvm/IR/Function.h"
71 #include "llvm/IR/InlineAsm.h"
72 #include "llvm/IR/InstrTypes.h"
73 #include "llvm/IR/Instructions.h"
74 #include "llvm/IR/IntrinsicInst.h"
75 #include "llvm/IR/Intrinsics.h"
76 #include "llvm/IR/IntrinsicsAArch64.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
78 #include "llvm/IR/LLVMContext.h"
79 #include "llvm/IR/Metadata.h"
80 #include "llvm/IR/Module.h"
81 #include "llvm/IR/Operator.h"
82 #include "llvm/IR/PatternMatch.h"
83 #include "llvm/IR/Statepoint.h"
84 #include "llvm/IR/Type.h"
85 #include "llvm/IR/User.h"
86 #include "llvm/IR/Value.h"
87 #include "llvm/MC/MCContext.h"
89 #include "llvm/Support/Casting.h"
91 #include "llvm/Support/Compiler.h"
92 #include "llvm/Support/Debug.h"
99 #include <cstddef>
100 #include <iterator>
101 #include <limits>
102 #include <optional>
103 #include <tuple>
104 
105 using namespace llvm;
106 using namespace PatternMatch;
107 using namespace SwitchCG;
108 
109 #define DEBUG_TYPE "isel"
110 
111 /// LimitFloatPrecision - Generate low-precision inline sequences for
112 /// some float libcalls (6, 8 or 12 bits).
113 static unsigned LimitFloatPrecision;
114 
115 static cl::opt<bool>
116  InsertAssertAlign("insert-assert-align", cl::init(true),
117  cl::desc("Insert the experimental `assertalign` node."),
119 
121  LimitFPPrecision("limit-float-precision",
122  cl::desc("Generate low-precision inline sequences "
123  "for some float libcalls"),
125  cl::init(0));
126 
128  "switch-peel-threshold", cl::Hidden, cl::init(66),
129  cl::desc("Set the case probability threshold for peeling the case from a "
130  "switch statement. A value greater than 100 will void this "
131  "optimization"));
132 
133 // Limit the width of DAG chains. This is important in general to prevent
134 // DAG-based analysis from blowing up. For example, alias analysis and
135 // load clustering may not complete in reasonable time. It is difficult to
136 // recognize and avoid this situation within each individual analysis, and
137 // future analyses are likely to have the same behavior. Limiting DAG width is
138 // the safe approach and will be especially important with global DAGs.
139 //
140 // MaxParallelChains default is arbitrarily high to avoid affecting
141 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
142 // sequence over this should have been converted to llvm.memcpy by the
143 // frontend. It is easy to induce this behavior with .ll code such as:
144 // %buffer = alloca [4096 x i8]
145 // %data = load [4096 x i8]* %argPtr
146 // store [4096 x i8] %data, [4096 x i8]* %buffer
147 static const unsigned MaxParallelChains = 64;
148 
150  const SDValue *Parts, unsigned NumParts,
151  MVT PartVT, EVT ValueVT, const Value *V,
152  std::optional<CallingConv::ID> CC);
153 
154 /// getCopyFromParts - Create a value that contains the specified legal parts
155 /// combined into the value they represent. If the parts combine to a type
156 /// larger than ValueVT then AssertOp can be used to specify whether the extra
157 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
158 /// (ISD::AssertSext).
159 static SDValue
160 getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
161  unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
162  std::optional<CallingConv::ID> CC = std::nullopt,
163  std::optional<ISD::NodeType> AssertOp = std::nullopt) {
164  // Let the target assemble the parts if it wants to
165  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
166  if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
167  PartVT, ValueVT, CC))
168  return Val;
169 
170  if (ValueVT.isVector())
171  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
172  CC);
173 
174  assert(NumParts > 0 && "No parts to assemble!");
175  SDValue Val = Parts[0];
176 
177  if (NumParts > 1) {
178  // Assemble the value from multiple parts.
179  if (ValueVT.isInteger()) {
180  unsigned PartBits = PartVT.getSizeInBits();
181  unsigned ValueBits = ValueVT.getSizeInBits();
182 
183  // Assemble the power of 2 part.
184  unsigned RoundParts =
185  (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
186  unsigned RoundBits = PartBits * RoundParts;
187  EVT RoundVT = RoundBits == ValueBits ?
188  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
189  SDValue Lo, Hi;
190 
191  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
192 
193  if (RoundParts > 2) {
194  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
195  PartVT, HalfVT, V);
196  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
197  RoundParts / 2, PartVT, HalfVT, V);
198  } else {
199  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
200  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
201  }
202 
203  if (DAG.getDataLayout().isBigEndian())
204  std::swap(Lo, Hi);
205 
206  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
207 
208  if (RoundParts < NumParts) {
209  // Assemble the trailing non-power-of-2 part.
210  unsigned OddParts = NumParts - RoundParts;
211  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
212  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
213  OddVT, V, CC);
214 
215  // Combine the round and odd parts.
216  Lo = Val;
217  if (DAG.getDataLayout().isBigEndian())
218  std::swap(Lo, Hi);
219  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
220  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
221  Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
222  DAG.getConstant(Lo.getValueSizeInBits(), DL,
223  TLI.getShiftAmountTy(
224  TotalVT, DAG.getDataLayout())));
225  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
226  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
227  }
228  } else if (PartVT.isFloatingPoint()) {
229  // FP split into multiple FP parts (for ppcf128)
230  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
231  "Unexpected split");
232  SDValue Lo, Hi;
233  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
234  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
235  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
236  std::swap(Lo, Hi);
237  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
238  } else {
239  // FP split into integer parts (soft fp)
240  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
241  !PartVT.isVector() && "Unexpected split");
242  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
243  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
244  }
245  }
246 
247  // There is now one part, held in Val. Correct it to match ValueVT.
248  // PartEVT is the type of the register class that holds the value.
249  // ValueVT is the type of the inline asm operation.
250  EVT PartEVT = Val.getValueType();
251 
252  if (PartEVT == ValueVT)
253  return Val;
254 
255  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
256  ValueVT.bitsLT(PartEVT)) {
257  // For an FP value in an integer part, we need to truncate to the right
258  // width first.
259  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
260  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
261  }
262 
263  // Handle types that have the same size.
264  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
265  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
266 
267  // Handle types with different sizes.
268  if (PartEVT.isInteger() && ValueVT.isInteger()) {
269  if (ValueVT.bitsLT(PartEVT)) {
270  // For a truncate, see if we have any information to
271  // indicate whether the truncated bits will always be
272  // zero or sign-extension.
273  if (AssertOp)
274  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
275  DAG.getValueType(ValueVT));
276  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
277  }
278  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
279  }
280 
281  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
282  // FP_ROUND's are always exact here.
283  if (ValueVT.bitsLT(Val.getValueType()))
284  return DAG.getNode(
285  ISD::FP_ROUND, DL, ValueVT, Val,
286  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
287 
288  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
289  }
290 
291  // Handle MMX to a narrower integer type by bitcasting MMX to integer and
292  // then truncating.
293  if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
294  ValueVT.bitsLT(PartEVT)) {
295  Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
296  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
297  }
298 
299  report_fatal_error("Unknown mismatch in getCopyFromParts!");
300 }
301 
303  const Twine &ErrMsg) {
304  const Instruction *I = dyn_cast_or_null<Instruction>(V);
305  if (!V)
306  return Ctx.emitError(ErrMsg);
307 
308  const char *AsmError = ", possible invalid constraint for vector type";
309  if (const CallInst *CI = dyn_cast<CallInst>(I))
310  if (CI->isInlineAsm())
311  return Ctx.emitError(I, ErrMsg + AsmError);
312 
313  return Ctx.emitError(I, ErrMsg);
314 }
315 
316 /// getCopyFromPartsVector - Create a value that contains the specified legal
317 /// parts combined into the value they represent. If the parts combine to a
318 /// type larger than ValueVT then AssertOp can be used to specify whether the
319 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
320 /// ValueVT (ISD::AssertSext).
322  const SDValue *Parts, unsigned NumParts,
323  MVT PartVT, EVT ValueVT, const Value *V,
324  std::optional<CallingConv::ID> CallConv) {
325  assert(ValueVT.isVector() && "Not a vector value");
326  assert(NumParts > 0 && "No parts to assemble!");
327  const bool IsABIRegCopy = CallConv.has_value();
328 
329  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
330  SDValue Val = Parts[0];
331 
332  // Handle a multi-element vector.
333  if (NumParts > 1) {
334  EVT IntermediateVT;
335  MVT RegisterVT;
336  unsigned NumIntermediates;
337  unsigned NumRegs;
338 
339  if (IsABIRegCopy) {
341  *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
342  NumIntermediates, RegisterVT);
343  } else {
344  NumRegs =
345  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
346  NumIntermediates, RegisterVT);
347  }
348 
349  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
350  NumParts = NumRegs; // Silence a compiler warning.
351  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
352  assert(RegisterVT.getSizeInBits() ==
353  Parts[0].getSimpleValueType().getSizeInBits() &&
354  "Part type sizes don't match!");
355 
356  // Assemble the parts into intermediate operands.
357  SmallVector<SDValue, 8> Ops(NumIntermediates);
358  if (NumIntermediates == NumParts) {
359  // If the register was not expanded, truncate or copy the value,
360  // as appropriate.
361  for (unsigned i = 0; i != NumParts; ++i)
362  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
363  PartVT, IntermediateVT, V, CallConv);
364  } else if (NumParts > 0) {
365  // If the intermediate type was expanded, build the intermediate
366  // operands from the parts.
367  assert(NumParts % NumIntermediates == 0 &&
368  "Must expand into a divisible number of parts!");
369  unsigned Factor = NumParts / NumIntermediates;
370  for (unsigned i = 0; i != NumIntermediates; ++i)
371  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
372  PartVT, IntermediateVT, V, CallConv);
373  }
374 
375  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
376  // intermediate operands.
377  EVT BuiltVectorTy =
378  IntermediateVT.isVector()
380  *DAG.getContext(), IntermediateVT.getScalarType(),
381  IntermediateVT.getVectorElementCount() * NumParts)
382  : EVT::getVectorVT(*DAG.getContext(),
383  IntermediateVT.getScalarType(),
384  NumIntermediates);
385  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
387  DL, BuiltVectorTy, Ops);
388  }
389 
390  // There is now one part, held in Val. Correct it to match ValueVT.
391  EVT PartEVT = Val.getValueType();
392 
393  if (PartEVT == ValueVT)
394  return Val;
395 
396  if (PartEVT.isVector()) {
397  // Vector/Vector bitcast.
398  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
399  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
400 
401  // If the parts vector has more elements than the value vector, then we
402  // have a vector widening case (e.g. <2 x float> -> <4 x float>).
403  // Extract the elements we want.
404  if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
407  (PartEVT.getVectorElementCount().isScalable() ==
408  ValueVT.getVectorElementCount().isScalable()) &&
409  "Cannot narrow, it would be a lossy transformation");
410  PartEVT =
412  ValueVT.getVectorElementCount());
413  Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
414  DAG.getVectorIdxConstant(0, DL));
415  if (PartEVT == ValueVT)
416  return Val;
417  if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
418  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
419  }
420 
421  // Promoted vector extract
422  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
423  }
424 
425  // Trivial bitcast if the types are the same size and the destination
426  // vector type is legal.
427  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
428  TLI.isTypeLegal(ValueVT))
429  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
430 
431  if (ValueVT.getVectorNumElements() != 1) {
432  // Certain ABIs require that vectors are passed as integers. For vectors
433  // are the same size, this is an obvious bitcast.
434  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
435  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
436  } else if (ValueVT.bitsLT(PartEVT)) {
437  const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
438  EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
439  // Drop the extra bits.
440  Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
441  return DAG.getBitcast(ValueVT, Val);
442  }
443 
445  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
446  return DAG.getUNDEF(ValueVT);
447  }
448 
449  // Handle cases such as i8 -> <1 x i1>
450  EVT ValueSVT = ValueVT.getVectorElementType();
451  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
452  unsigned ValueSize = ValueSVT.getSizeInBits();
453  if (ValueSize == PartEVT.getSizeInBits()) {
454  Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
455  } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
456  // It's possible a scalar floating point type gets softened to integer and
457  // then promoted to a larger integer. If PartEVT is the larger integer
458  // we need to truncate it and then bitcast to the FP type.
459  assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
460  EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
461  Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
462  Val = DAG.getBitcast(ValueSVT, Val);
463  } else {
464  Val = ValueVT.isFloatingPoint()
465  ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
466  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
467  }
468  }
469 
470  return DAG.getBuildVector(ValueVT, DL, Val);
471 }
472 
473 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
474  SDValue Val, SDValue *Parts, unsigned NumParts,
475  MVT PartVT, const Value *V,
476  std::optional<CallingConv::ID> CallConv);
477 
478 /// getCopyToParts - Create a series of nodes that contain the specified value
479 /// split into legal parts. If the parts contain more bits than Val, then, for
480 /// integers, ExtendKind can be used to specify how to generate the extra bits.
481 static void
482 getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
483  unsigned NumParts, MVT PartVT, const Value *V,
484  std::optional<CallingConv::ID> CallConv = std::nullopt,
485  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
486  // Let the target split the parts if it wants to
487  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
488  if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
489  CallConv))
490  return;
491  EVT ValueVT = Val.getValueType();
492 
493  // Handle the vector case separately.
494  if (ValueVT.isVector())
495  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
496  CallConv);
497 
498  unsigned PartBits = PartVT.getSizeInBits();
499  unsigned OrigNumParts = NumParts;
500  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
501  "Copying to an illegal type!");
502 
503  if (NumParts == 0)
504  return;
505 
506  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
507  EVT PartEVT = PartVT;
508  if (PartEVT == ValueVT) {
509  assert(NumParts == 1 && "No-op copy with multiple parts!");
510  Parts[0] = Val;
511  return;
512  }
513 
514  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
515  // If the parts cover more bits than the value has, promote the value.
516  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
517  assert(NumParts == 1 && "Do not know what to promote to!");
518  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
519  } else {
520  if (ValueVT.isFloatingPoint()) {
521  // FP values need to be bitcast, then extended if they are being put
522  // into a larger container.
523  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
524  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
525  }
526  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
527  ValueVT.isInteger() &&
528  "Unknown mismatch!");
529  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
530  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
531  if (PartVT == MVT::x86mmx)
532  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533  }
534  } else if (PartBits == ValueVT.getSizeInBits()) {
535  // Different types of the same size.
536  assert(NumParts == 1 && PartEVT != ValueVT);
537  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
538  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
539  // If the parts cover less bits than value has, truncate the value.
540  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
541  ValueVT.isInteger() &&
542  "Unknown mismatch!");
543  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
544  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
545  if (PartVT == MVT::x86mmx)
546  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547  }
548 
549  // The value may have changed - recompute ValueVT.
550  ValueVT = Val.getValueType();
551  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
552  "Failed to tile the value with PartVT!");
553 
554  if (NumParts == 1) {
555  if (PartEVT != ValueVT) {
557  "scalar-to-vector conversion failed");
558  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559  }
560 
561  Parts[0] = Val;
562  return;
563  }
564 
565  // Expand the value into multiple parts.
566  if (NumParts & (NumParts - 1)) {
567  // The number of parts is not a power of 2. Split off and copy the tail.
568  assert(PartVT.isInteger() && ValueVT.isInteger() &&
569  "Do not know what to expand to!");
570  unsigned RoundParts = 1 << Log2_32(NumParts);
571  unsigned RoundBits = RoundParts * PartBits;
572  unsigned OddParts = NumParts - RoundParts;
573  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
574  DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
575 
576  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
577  CallConv);
578 
579  if (DAG.getDataLayout().isBigEndian())
580  // The odd parts were reversed by getCopyToParts - unreverse them.
581  std::reverse(Parts + RoundParts, Parts + NumParts);
582 
583  NumParts = RoundParts;
584  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
585  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
586  }
587 
588  // The number of parts is a power of 2. Repeatedly bisect the value using
589  // EXTRACT_ELEMENT.
590  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
592  ValueVT.getSizeInBits()),
593  Val);
594 
595  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
596  for (unsigned i = 0; i < NumParts; i += StepSize) {
597  unsigned ThisBits = StepSize * PartBits / 2;
598  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
599  SDValue &Part0 = Parts[i];
600  SDValue &Part1 = Parts[i+StepSize/2];
601 
602  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
603  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
604  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
605  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
606 
607  if (ThisBits == PartBits && ThisVT != PartVT) {
608  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
609  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
610  }
611  }
612  }
613 
614  if (DAG.getDataLayout().isBigEndian())
615  std::reverse(Parts, Parts + OrigNumParts);
616 }
617 
619  const SDLoc &DL, EVT PartVT) {
620  if (!PartVT.isVector())
621  return SDValue();
622 
623  EVT ValueVT = Val.getValueType();
624  ElementCount PartNumElts = PartVT.getVectorElementCount();
625  ElementCount ValueNumElts = ValueVT.getVectorElementCount();
626 
627  // We only support widening vectors with equivalent element types and
628  // fixed/scalable properties. If a target needs to widen a fixed-length type
629  // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
630  if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
631  PartNumElts.isScalable() != ValueNumElts.isScalable() ||
632  PartVT.getVectorElementType() != ValueVT.getVectorElementType())
633  return SDValue();
634 
635  // Widening a scalable vector to another scalable vector is done by inserting
636  // the vector into a larger undef one.
637  if (PartNumElts.isScalable())
638  return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
639  Val, DAG.getVectorIdxConstant(0, DL));
640 
641  EVT ElementVT = PartVT.getVectorElementType();
642  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
643  // undef elements.
645  DAG.ExtractVectorElements(Val, Ops);
646  SDValue EltUndef = DAG.getUNDEF(ElementVT);
647  Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
648 
649  // FIXME: Use CONCAT for 2x -> 4x.
650  return DAG.getBuildVector(PartVT, DL, Ops);
651 }
652 
653 /// getCopyToPartsVector - Create a series of nodes that contain the specified
654 /// value split into legal parts.
655 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
656  SDValue Val, SDValue *Parts, unsigned NumParts,
657  MVT PartVT, const Value *V,
658  std::optional<CallingConv::ID> CallConv) {
659  EVT ValueVT = Val.getValueType();
660  assert(ValueVT.isVector() && "Not a vector");
661  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
662  const bool IsABIRegCopy = CallConv.has_value();
663 
664  if (NumParts == 1) {
665  EVT PartEVT = PartVT;
666  if (PartEVT == ValueVT) {
667  // Nothing to do.
668  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
669  // Bitconvert vector->vector case.
670  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
671  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
672  Val = Widened;
673  } else if (PartVT.isVector() &&
674  PartEVT.getVectorElementType().bitsGE(
675  ValueVT.getVectorElementType()) &&
676  PartEVT.getVectorElementCount() ==
677  ValueVT.getVectorElementCount()) {
678 
679  // Promoted vector extract
680  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
681  } else if (PartEVT.isVector() &&
682  PartEVT.getVectorElementType() !=
683  ValueVT.getVectorElementType() &&
684  TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
686  // Combination of widening and promotion.
687  EVT WidenVT =
689  PartVT.getVectorElementCount());
690  SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
691  Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
692  } else {
693  // Don't extract an integer from a float vector. This can happen if the
694  // FP type gets softened to integer and then promoted. The promotion
695  // prevents it from being picked up by the earlier bitcast case.
696  if (ValueVT.getVectorElementCount().isScalar() &&
697  (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
698  Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
699  DAG.getVectorIdxConstant(0, DL));
700  } else {
701  uint64_t ValueSize = ValueVT.getFixedSizeInBits();
702  assert(PartVT.getFixedSizeInBits() > ValueSize &&
703  "lossy conversion of vector to scalar type");
704  EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
705  Val = DAG.getBitcast(IntermediateType, Val);
706  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
707  }
708  }
709 
710  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
711  Parts[0] = Val;
712  return;
713  }
714 
715  // Handle a multi-element vector.
716  EVT IntermediateVT;
717  MVT RegisterVT;
718  unsigned NumIntermediates;
719  unsigned NumRegs;
720  if (IsABIRegCopy) {
722  *DAG.getContext(), CallConv.value(), ValueVT, IntermediateVT,
723  NumIntermediates, RegisterVT);
724  } else {
725  NumRegs =
726  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
727  NumIntermediates, RegisterVT);
728  }
729 
730  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
731  NumParts = NumRegs; // Silence a compiler warning.
732  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
733 
734  assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
735  "Mixing scalable and fixed vectors when copying in parts");
736 
737  std::optional<ElementCount> DestEltCnt;
738 
739  if (IntermediateVT.isVector())
740  DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
741  else
742  DestEltCnt = ElementCount::getFixed(NumIntermediates);
743 
744  EVT BuiltVectorTy = EVT::getVectorVT(
745  *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
746 
747  if (ValueVT == BuiltVectorTy) {
748  // Nothing to do.
749  } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
750  // Bitconvert vector->vector case.
751  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
752  } else {
753  if (BuiltVectorTy.getVectorElementType().bitsGT(
754  ValueVT.getVectorElementType())) {
755  // Integer promotion.
756  ValueVT = EVT::getVectorVT(*DAG.getContext(),
757  BuiltVectorTy.getVectorElementType(),
758  ValueVT.getVectorElementCount());
759  Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
760  }
761 
762  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
763  Val = Widened;
764  }
765  }
766 
767  assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
768 
769  // Split the vector into intermediate operands.
770  SmallVector<SDValue, 8> Ops(NumIntermediates);
771  for (unsigned i = 0; i != NumIntermediates; ++i) {
772  if (IntermediateVT.isVector()) {
773  // This does something sensible for scalable vectors - see the
774  // definition of EXTRACT_SUBVECTOR for further details.
775  unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
776  Ops[i] =
777  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
778  DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
779  } else {
780  Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
781  DAG.getVectorIdxConstant(i, DL));
782  }
783  }
784 
785  // Split the intermediate operands into legal parts.
786  if (NumParts == NumIntermediates) {
787  // If the register was not expanded, promote or copy the value,
788  // as appropriate.
789  for (unsigned i = 0; i != NumParts; ++i)
790  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
791  } else if (NumParts > 0) {
792  // If the intermediate type was expanded, split each the value into
793  // legal parts.
794  assert(NumIntermediates != 0 && "division by zero");
795  assert(NumParts % NumIntermediates == 0 &&
796  "Must expand into a divisible number of parts!");
797  unsigned Factor = NumParts / NumIntermediates;
798  for (unsigned i = 0; i != NumIntermediates; ++i)
799  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
800  CallConv);
801  }
802 }
803 
805  EVT valuevt, std::optional<CallingConv::ID> CC)
806  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
807  RegCount(1, regs.size()), CallConv(CC) {}
808 
810  const DataLayout &DL, unsigned Reg, Type *Ty,
811  std::optional<CallingConv::ID> CC) {
812  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
813 
814  CallConv = CC;
815 
816  for (EVT ValueVT : ValueVTs) {
817  unsigned NumRegs =
818  isABIMangled()
819  ? TLI.getNumRegistersForCallingConv(Context, CC.value(), ValueVT)
820  : TLI.getNumRegisters(Context, ValueVT);
821  MVT RegisterVT =
822  isABIMangled()
823  ? TLI.getRegisterTypeForCallingConv(Context, CC.value(), ValueVT)
824  : TLI.getRegisterType(Context, ValueVT);
825  for (unsigned i = 0; i != NumRegs; ++i)
826  Regs.push_back(Reg + i);
827  RegVTs.push_back(RegisterVT);
828  RegCount.push_back(NumRegs);
829  Reg += NumRegs;
830  }
831 }
832 
834  FunctionLoweringInfo &FuncInfo,
835  const SDLoc &dl, SDValue &Chain,
836  SDValue *Flag, const Value *V) const {
837  // A Value with type {} or [0 x %t] needs no registers.
838  if (ValueVTs.empty())
839  return SDValue();
840 
841  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
842 
843  // Assemble the legal parts into the final values.
844  SmallVector<SDValue, 4> Values(ValueVTs.size());
846  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
847  // Copy the legal parts from the registers.
848  EVT ValueVT = ValueVTs[Value];
849  unsigned NumRegs = RegCount[Value];
850  MVT RegisterVT =
852  *DAG.getContext(), CallConv.value(), RegVTs[Value])
853  : RegVTs[Value];
854 
855  Parts.resize(NumRegs);
856  for (unsigned i = 0; i != NumRegs; ++i) {
857  SDValue P;
858  if (!Flag) {
859  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
860  } else {
861  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
862  *Flag = P.getValue(2);
863  }
864 
865  Chain = P.getValue(1);
866  Parts[i] = P;
867 
868  // If the source register was virtual and if we know something about it,
869  // add an assert node.
870  if (!Register::isVirtualRegister(Regs[Part + i]) ||
871  !RegisterVT.isInteger())
872  continue;
873 
875  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
876  if (!LOI)
877  continue;
878 
879  unsigned RegSize = RegisterVT.getScalarSizeInBits();
880  unsigned NumSignBits = LOI->NumSignBits;
881  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
882 
883  if (NumZeroBits == RegSize) {
884  // The current value is a zero.
885  // Explicitly express that as it would be easier for
886  // optimizations to kick in.
887  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
888  continue;
889  }
890 
891  // FIXME: We capture more information than the dag can represent. For
892  // now, just use the tightest assertzext/assertsext possible.
893  bool isSExt;
894  EVT FromVT(MVT::Other);
895  if (NumZeroBits) {
896  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
897  isSExt = false;
898  } else if (NumSignBits > 1) {
899  FromVT =
900  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
901  isSExt = true;
902  } else {
903  continue;
904  }
905  // Add an assertion node.
906  assert(FromVT != MVT::Other);
907  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
908  RegisterVT, P, DAG.getValueType(FromVT));
909  }
910 
911  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
912  RegisterVT, ValueVT, V, CallConv);
913  Part += NumRegs;
914  Parts.clear();
915  }
916 
917  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
918 }
919 
921  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
922  const Value *V,
923  ISD::NodeType PreferredExtendType) const {
924  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
925  ISD::NodeType ExtendKind = PreferredExtendType;
926 
927  // Get the list of the values's legal parts.
928  unsigned NumRegs = Regs.size();
929  SmallVector<SDValue, 8> Parts(NumRegs);
930  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
931  unsigned NumParts = RegCount[Value];
932 
933  MVT RegisterVT =
935  *DAG.getContext(), CallConv.value(), RegVTs[Value])
936  : RegVTs[Value];
937 
938  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
939  ExtendKind = ISD::ZERO_EXTEND;
940 
941  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
942  NumParts, RegisterVT, V, CallConv, ExtendKind);
943  Part += NumParts;
944  }
945 
946  // Copy the parts into the registers.
947  SmallVector<SDValue, 8> Chains(NumRegs);
948  for (unsigned i = 0; i != NumRegs; ++i) {
949  SDValue Part;
950  if (!Flag) {
951  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
952  } else {
953  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
954  *Flag = Part.getValue(1);
955  }
956 
957  Chains[i] = Part.getValue(0);
958  }
959 
960  if (NumRegs == 1 || Flag)
961  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
962  // flagged to it. That is the CopyToReg nodes and the user are considered
963  // a single scheduling unit. If we create a TokenFactor and return it as
964  // chain, then the TokenFactor is both a predecessor (operand) of the
965  // user as well as a successor (the TF operands are flagged to the user).
966  // c1, f1 = CopyToReg
967  // c2, f2 = CopyToReg
968  // c3 = TokenFactor c1, c2
969  // ...
970  // = op c3, ..., f2
971  Chain = Chains[NumRegs-1];
972  else
973  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
974 }
975 
976 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
977  unsigned MatchingIdx, const SDLoc &dl,
978  SelectionDAG &DAG,
979  std::vector<SDValue> &Ops) const {
980  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
981 
982  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
983  if (HasMatching)
985  else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
986  // Put the register class of the virtual registers in the flag word. That
987  // way, later passes can recompute register class constraints for inline
988  // assembly as well as normal instructions.
989  // Don't do this for tied operands that can use the regclass information
990  // from the def.
992  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
994  }
995 
996  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
997  Ops.push_back(Res);
998 
999  if (Code == InlineAsm::Kind_Clobber) {
1000  // Clobbers should always have a 1:1 mapping with registers, and may
1001  // reference registers that have illegal (e.g. vector) types. Hence, we
1002  // shouldn't try to apply any sort of splitting logic to them.
1003  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1004  "No 1:1 mapping from clobbers to regs?");
1006  (void)SP;
1007  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1008  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1009  assert(
1010  (Regs[I] != SP ||
1012  "If we clobbered the stack pointer, MFI should know about it.");
1013  }
1014  return;
1015  }
1016 
1017  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1018  MVT RegisterVT = RegVTs[Value];
1019  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1020  RegisterVT);
1021  for (unsigned i = 0; i != NumRegs; ++i) {
1022  assert(Reg < Regs.size() && "Mismatch in # registers expected");
1023  unsigned TheReg = Regs[Reg++];
1024  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1025  }
1026  }
1027 }
1028 
1032  unsigned I = 0;
1033  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1034  unsigned RegCount = std::get<0>(CountAndVT);
1035  MVT RegisterVT = std::get<1>(CountAndVT);
1036  TypeSize RegisterSize = RegisterVT.getSizeInBits();
1037  for (unsigned E = I + RegCount; I != E; ++I)
1038  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1039  }
1040  return OutVec;
1041 }
1042 
1044  AssumptionCache *ac,
1045  const TargetLibraryInfo *li) {
1046  AA = aa;
1047  AC = ac;
1048  GFI = gfi;
1049  LibInfo = li;
1050  Context = DAG.getContext();
1051  LPadToCallSiteMap.clear();
1052  SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1053 }
1054 
1056  NodeMap.clear();
1057  UnusedArgNodeMap.clear();
1058  PendingLoads.clear();
1059  PendingExports.clear();
1060  PendingConstrainedFP.clear();
1061  PendingConstrainedFPStrict.clear();
1062  CurInst = nullptr;
1063  HasTailCall = false;
1064  SDNodeOrder = LowestSDNodeOrder;
1066 }
1067 
1069  DanglingDebugInfoMap.clear();
1070 }
1071 
1072 // Update DAG root to include dependencies on Pending chains.
1073 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1074  SDValue Root = DAG.getRoot();
1075 
1076  if (Pending.empty())
1077  return Root;
1078 
1079  // Add current root to PendingChains, unless we already indirectly
1080  // depend on it.
1081  if (Root.getOpcode() != ISD::EntryToken) {
1082  unsigned i = 0, e = Pending.size();
1083  for (; i != e; ++i) {
1084  assert(Pending[i].getNode()->getNumOperands() > 1);
1085  if (Pending[i].getNode()->getOperand(0) == Root)
1086  break; // Don't add the root if we already indirectly depend on it.
1087  }
1088 
1089  if (i == e)
1090  Pending.push_back(Root);
1091  }
1092 
1093  if (Pending.size() == 1)
1094  Root = Pending[0];
1095  else
1096  Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1097 
1098  DAG.setRoot(Root);
1099  Pending.clear();
1100  return Root;
1101 }
1102 
1104  return updateRoot(PendingLoads);
1105 }
1106 
1108  // Chain up all pending constrained intrinsics together with all
1109  // pending loads, by simply appending them to PendingLoads and
1110  // then calling getMemoryRoot().
1111  PendingLoads.reserve(PendingLoads.size() +
1112  PendingConstrainedFP.size() +
1113  PendingConstrainedFPStrict.size());
1114  PendingLoads.append(PendingConstrainedFP.begin(),
1115  PendingConstrainedFP.end());
1116  PendingLoads.append(PendingConstrainedFPStrict.begin(),
1117  PendingConstrainedFPStrict.end());
1118  PendingConstrainedFP.clear();
1119  PendingConstrainedFPStrict.clear();
1120  return getMemoryRoot();
1121 }
1122 
1124  // We need to emit pending fpexcept.strict constrained intrinsics,
1125  // so append them to the PendingExports list.
1126  PendingExports.append(PendingConstrainedFPStrict.begin(),
1127  PendingConstrainedFPStrict.end());
1128  PendingConstrainedFPStrict.clear();
1129  return updateRoot(PendingExports);
1130 }
1131 
1133  // Set up outgoing PHI node register values before emitting the terminator.
1134  if (I.isTerminator()) {
1135  HandlePHINodesInSuccessorBlocks(I.getParent());
1136  }
1137 
1138  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1139  if (!isa<DbgInfoIntrinsic>(I))
1140  ++SDNodeOrder;
1141 
1142  CurInst = &I;
1143 
1144  // Set inserted listener only if required.
1145  bool NodeInserted = false;
1146  std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1147  MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1148  if (PCSectionsMD) {
1149  InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1150  DAG, [&](SDNode *) { NodeInserted = true; });
1151  }
1152 
1153  visit(I.getOpcode(), I);
1154 
1155  if (!I.isTerminator() && !HasTailCall &&
1156  !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1158 
1159  // Handle metadata.
1160  if (PCSectionsMD) {
1161  auto It = NodeMap.find(&I);
1162  if (It != NodeMap.end()) {
1163  DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1164  } else if (NodeInserted) {
1165  // This should not happen; if it does, don't let it go unnoticed so we can
1166  // fix it. Relevant visit*() function is probably missing a setValue().
1167  errs() << "warning: loosing !pcsections metadata ["
1168  << I.getModule()->getName() << "]\n";
1169  LLVM_DEBUG(I.dump());
1170  assert(false);
1171  }
1172  }
1173 
1174  CurInst = nullptr;
1175 }
1176 
1177 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1178  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1179 }
1180 
1181 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1182  // Note: this doesn't use InstVisitor, because it has to work with
1183  // ConstantExpr's in addition to instructions.
1184  switch (Opcode) {
1185  default: llvm_unreachable("Unknown instruction type encountered!");
1186  // Build the switch statement using the Instruction.def file.
1187 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1188  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1189 #include "llvm/IR/Instruction.def"
1190  }
1191 }
1192 
1194  unsigned Order) {
1195  // We treat variadic dbg_values differently at this stage.
1196  if (DI->hasArgList()) {
1197  // For variadic dbg_values we will now insert an undef.
1198  // FIXME: We can potentially recover these!
1200  for (const Value *V : DI->getValues()) {
1201  auto Undef = UndefValue::get(V->getType());
1202  Locs.push_back(SDDbgOperand::fromConst(Undef));
1203  }
1205  DI->getVariable(), DI->getExpression(), Locs, {},
1206  /*IsIndirect=*/false, DI->getDebugLoc(), Order, /*IsVariadic=*/true);
1207  DAG.AddDbgValue(SDV, /*isParameter=*/false);
1208  } else {
1209  // TODO: Dangling debug info will eventually either be resolved or produce
1210  // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1211  // between the original dbg.value location and its resolved DBG_VALUE,
1212  // which we should ideally fill with an extra Undef DBG_VALUE.
1213  assert(DI->getNumVariableLocationOps() == 1 &&
1214  "DbgValueInst without an ArgList should have a single location "
1215  "operand.");
1216  DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);
1217  }
1218 }
1219 
1221  const DIExpression *Expr) {
1222  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1223  DIVariable *DanglingVariable = DDI.getVariable();
1224  DIExpression *DanglingExpr = DDI.getExpression();
1225  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1226  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << DDI << "\n");
1227  return true;
1228  }
1229  return false;
1230  };
1231 
1232  for (auto &DDIMI : DanglingDebugInfoMap) {
1233  DanglingDebugInfoVector &DDIV = DDIMI.second;
1234 
1235  // If debug info is to be dropped, run it through final checks to see
1236  // whether it can be salvaged.
1237  for (auto &DDI : DDIV)
1238  if (isMatchingDbgValue(DDI))
1240 
1241  erase_if(DDIV, isMatchingDbgValue);
1242  }
1243 }
1244 
1245 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1246 // generate the debug data structures now that we've seen its definition.
1248  SDValue Val) {
1249  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1250  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1251  return;
1252 
1253  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1254  for (auto &DDI : DDIV) {
1255  DebugLoc DL = DDI.getDebugLoc();
1256  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1257  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1258  DILocalVariable *Variable = DDI.getVariable();
1259  DIExpression *Expr = DDI.getExpression();
1260  assert(Variable->isValidLocationForIntrinsic(DL) &&
1261  "Expected inlined-at fields to agree");
1262  SDDbgValue *SDV;
1263  if (Val.getNode()) {
1264  // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1265  // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1266  // we couldn't resolve it directly when examining the DbgValue intrinsic
1267  // in the first place we should not be more successful here). Unless we
1268  // have some test case that prove this to be correct we should avoid
1269  // calling EmitFuncArgumentDbgValue here.
1270  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1271  FuncArgumentDbgValueKind::Value, Val)) {
1272  LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << DDI << "\n");
1273  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1274  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1275  // inserted after the definition of Val when emitting the instructions
1276  // after ISel. An alternative could be to teach
1277  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1278  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1279  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1280  << ValSDNodeOrder << "\n");
1281  SDV = getDbgValue(Val, Variable, Expr, DL,
1282  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1283  DAG.AddDbgValue(SDV, false);
1284  } else
1285  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << DDI
1286  << "in EmitFuncArgumentDbgValue\n");
1287  } else {
1288  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DDI << "\n");
1289  auto Undef = UndefValue::get(V->getType());
1290  auto SDV =
1291  DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1292  DAG.AddDbgValue(SDV, false);
1293  }
1294  }
1295  DDIV.clear();
1296 }
1297 
1299  // TODO: For the variadic implementation, instead of only checking the fail
1300  // state of `handleDebugValue`, we need know specifically which values were
1301  // invalid, so that we attempt to salvage only those values when processing
1302  // a DIArgList.
1303  Value *V = DDI.getVariableLocationOp(0);
1304  Value *OrigV = V;
1305  DILocalVariable *Var = DDI.getVariable();
1306  DIExpression *Expr = DDI.getExpression();
1307  DebugLoc DL = DDI.getDebugLoc();
1308  unsigned SDOrder = DDI.getSDNodeOrder();
1309 
1310  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1311  // that DW_OP_stack_value is desired.
1312  bool StackValue = true;
1313 
1314  // Can this Value can be encoded without any further work?
1315  if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1316  return;
1317 
1318  // Attempt to salvage back through as many instructions as possible. Bail if
1319  // a non-instruction is seen, such as a constant expression or global
1320  // variable. FIXME: Further work could recover those too.
1321  while (isa<Instruction>(V)) {
1322  Instruction &VAsInst = *cast<Instruction>(V);
1323  // Temporary "0", awaiting real implementation.
1325  SmallVector<Value *, 4> AdditionalValues;
1326  V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1327  AdditionalValues);
1328  // If we cannot salvage any further, and haven't yet found a suitable debug
1329  // expression, bail out.
1330  if (!V)
1331  break;
1332 
1333  // TODO: If AdditionalValues isn't empty, then the salvage can only be
1334  // represented with a DBG_VALUE_LIST, so we give up. When we have support
1335  // here for variadic dbg_values, remove that condition.
1336  if (!AdditionalValues.empty())
1337  break;
1338 
1339  // New value and expr now represent this debuginfo.
1340  Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1341 
1342  // Some kind of simplification occurred: check whether the operand of the
1343  // salvaged debug expression can be encoded in this DAG.
1344  if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1345  LLVM_DEBUG(
1346  dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1347  << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1348  return;
1349  }
1350  }
1351 
1352  // This was the final opportunity to salvage this debug information, and it
1353  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1354  // any earlier variable location.
1355  assert(OrigV && "V shouldn't be null");
1356  auto *Undef = UndefValue::get(OrigV->getType());
1357  auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1358  DAG.AddDbgValue(SDV, false);
1359  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI << "\n");
1360 }
1361 
1363  DILocalVariable *Var,
1364  DIExpression *Expr, DebugLoc DbgLoc,
1365  unsigned Order, bool IsVariadic) {
1366  if (Values.empty())
1367  return true;
1368  SmallVector<SDDbgOperand> LocationOps;
1369  SmallVector<SDNode *> Dependencies;
1370  for (const Value *V : Values) {
1371  // Constant value.
1372  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1373  isa<ConstantPointerNull>(V)) {
1374  LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1375  continue;
1376  }
1377 
1378  // Look through IntToPtr constants.
1379  if (auto *CE = dyn_cast<ConstantExpr>(V))
1380  if (CE->getOpcode() == Instruction::IntToPtr) {
1381  LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1382  continue;
1383  }
1384 
1385  // If the Value is a frame index, we can create a FrameIndex debug value
1386  // without relying on the DAG at all.
1387  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1388  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1389  if (SI != FuncInfo.StaticAllocaMap.end()) {
1390  LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1391  continue;
1392  }
1393  }
1394 
1395  // Do not use getValue() in here; we don't want to generate code at
1396  // this point if it hasn't been done yet.
1397  SDValue N = NodeMap[V];
1398  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1399  N = UnusedArgNodeMap[V];
1400  if (N.getNode()) {
1401  // Only emit func arg dbg value for non-variadic dbg.values for now.
1402  if (!IsVariadic &&
1403  EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1404  FuncArgumentDbgValueKind::Value, N))
1405  return true;
1406  if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1407  // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1408  // describe stack slot locations.
1409  //
1410  // Consider "int x = 0; int *px = &x;". There are two kinds of
1411  // interesting debug values here after optimization:
1412  //
1413  // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1414  // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1415  //
1416  // Both describe the direct values of their associated variables.
1417  Dependencies.push_back(N.getNode());
1418  LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1419  continue;
1420  }
1421  LocationOps.emplace_back(
1422  SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1423  continue;
1424  }
1425 
1426  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1427  // Special rules apply for the first dbg.values of parameter variables in a
1428  // function. Identify them by the fact they reference Argument Values, that
1429  // they're parameters, and they are parameters of the current function. We
1430  // need to let them dangle until they get an SDNode.
1431  bool IsParamOfFunc =
1432  isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1433  if (IsParamOfFunc)
1434  return false;
1435 
1436  // The value is not used in this block yet (or it would have an SDNode).
1437  // We still want the value to appear for the user if possible -- if it has
1438  // an associated VReg, we can refer to that instead.
1439  auto VMI = FuncInfo.ValueMap.find(V);
1440  if (VMI != FuncInfo.ValueMap.end()) {
1441  unsigned Reg = VMI->second;
1442  // If this is a PHI node, it may be split up into several MI PHI nodes
1443  // (in FunctionLoweringInfo::set).
1444  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1445  V->getType(), std::nullopt);
1446  if (RFV.occupiesMultipleRegs()) {
1447  // FIXME: We could potentially support variadic dbg_values here.
1448  if (IsVariadic)
1449  return false;
1450  unsigned Offset = 0;
1451  unsigned BitsToDescribe = 0;
1452  if (auto VarSize = Var->getSizeInBits())
1453  BitsToDescribe = *VarSize;
1454  if (auto Fragment = Expr->getFragmentInfo())
1455  BitsToDescribe = Fragment->SizeInBits;
1456  for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1457  // Bail out if all bits are described already.
1458  if (Offset >= BitsToDescribe)
1459  break;
1460  // TODO: handle scalable vectors.
1461  unsigned RegisterSize = RegAndSize.second;
1462  unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1463  ? BitsToDescribe - Offset
1464  : RegisterSize;
1465  auto FragmentExpr = DIExpression::createFragmentExpression(
1466  Expr, Offset, FragmentSize);
1467  if (!FragmentExpr)
1468  continue;
1470  Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1471  DAG.AddDbgValue(SDV, false);
1472  Offset += RegisterSize;
1473  }
1474  return true;
1475  }
1476  // We can use simple vreg locations for variadic dbg_values as well.
1477  LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1478  continue;
1479  }
1480  // We failed to create a SDDbgOperand for V.
1481  return false;
1482  }
1483 
1484  // We have created a SDDbgOperand for each Value in Values.
1485  // Should use Order instead of SDNodeOrder?
1486  assert(!LocationOps.empty());
1487  SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1488  /*IsIndirect=*/false, DbgLoc,
1489  SDNodeOrder, IsVariadic);
1490  DAG.AddDbgValue(SDV, /*isParameter=*/false);
1491  return true;
1492 }
1493 
1495  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1496  for (auto &Pair : DanglingDebugInfoMap)
1497  for (auto &DDI : Pair.second)
1500 }
1501 
1502 /// getCopyFromRegs - If there was virtual register allocated for the value V
1503 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1506  SDValue Result;
1507 
1508  if (It != FuncInfo.ValueMap.end()) {
1509  Register InReg = It->second;
1510 
1512  DAG.getDataLayout(), InReg, Ty,
1513  std::nullopt); // This is not an ABI copy.
1514  SDValue Chain = DAG.getEntryNode();
1515  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1516  V);
1517  resolveDanglingDebugInfo(V, Result);
1518  }
1519 
1520  return Result;
1521 }
1522 
1523 /// getValue - Return an SDValue for the given Value.
1525  // If we already have an SDValue for this value, use it. It's important
1526  // to do this first, so that we don't create a CopyFromReg if we already
1527  // have a regular SDValue.
1528  SDValue &N = NodeMap[V];
1529  if (N.getNode()) return N;
1530 
1531  // If there's a virtual register allocated and initialized for this
1532  // value, use it.
1533  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1534  return copyFromReg;
1535 
1536  // Otherwise create a new SDValue and remember it.
1537  SDValue Val = getValueImpl(V);
1538  NodeMap[V] = Val;
1539  resolveDanglingDebugInfo(V, Val);
1540  return Val;
1541 }
1542 
1543 /// getNonRegisterValue - Return an SDValue for the given Value, but
1544 /// don't look in FuncInfo.ValueMap for a virtual register.
1546  // If we already have an SDValue for this value, use it.
1547  SDValue &N = NodeMap[V];
1548  if (N.getNode()) {
1549  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1550  // Remove the debug location from the node as the node is about to be used
1551  // in a location which may differ from the original debug location. This
1552  // is relevant to Constant and ConstantFP nodes because they can appear
1553  // as constant expressions inside PHI nodes.
1554  N->setDebugLoc(DebugLoc());
1555  }
1556  return N;
1557  }
1558 
1559  // Otherwise create a new SDValue and remember it.
1560  SDValue Val = getValueImpl(V);
1561  NodeMap[V] = Val;
1562  resolveDanglingDebugInfo(V, Val);
1563  return Val;
1564 }
1565 
1566 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1567 /// Create an SDValue for the given value.
1569  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1570 
1571  if (const Constant *C = dyn_cast<Constant>(V)) {
1572  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1573 
1574  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1575  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1576 
1577  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1578  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1579 
1580  if (isa<ConstantPointerNull>(C)) {
1581  unsigned AS = V->getType()->getPointerAddressSpace();
1582  return DAG.getConstant(0, getCurSDLoc(),
1583  TLI.getPointerTy(DAG.getDataLayout(), AS));
1584  }
1585 
1586  if (match(C, m_VScale(DAG.getDataLayout())))
1587  return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1588 
1589  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1590  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1591 
1592  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1593  return DAG.getUNDEF(VT);
1594 
1595  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1596  visit(CE->getOpcode(), *CE);
1597  SDValue N1 = NodeMap[V];
1598  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1599  return N1;
1600  }
1601 
1602  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1604  for (const Use &U : C->operands()) {
1605  SDNode *Val = getValue(U).getNode();
1606  // If the operand is an empty aggregate, there are no values.
1607  if (!Val) continue;
1608  // Add each leaf value from the operand to the Constants list
1609  // to form a flattened list of all the values.
1610  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1611  Constants.push_back(SDValue(Val, i));
1612  }
1613 
1615  }
1616 
1617  if (const ConstantDataSequential *CDS =
1618  dyn_cast<ConstantDataSequential>(C)) {
1620  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1621  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1622  // Add each leaf value from the operand to the Constants list
1623  // to form a flattened list of all the values.
1624  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1625  Ops.push_back(SDValue(Val, i));
1626  }
1627 
1628  if (isa<ArrayType>(CDS->getType()))
1629  return DAG.getMergeValues(Ops, getCurSDLoc());
1630  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1631  }
1632 
1633  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1634  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1635  "Unknown struct or array constant!");
1636 
1637  SmallVector<EVT, 4> ValueVTs;
1638  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1639  unsigned NumElts = ValueVTs.size();
1640  if (NumElts == 0)
1641  return SDValue(); // empty struct
1643  for (unsigned i = 0; i != NumElts; ++i) {
1644  EVT EltVT = ValueVTs[i];
1645  if (isa<UndefValue>(C))
1646  Constants[i] = DAG.getUNDEF(EltVT);
1647  else if (EltVT.isFloatingPoint())
1648  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1649  else
1650  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1651  }
1652 
1654  }
1655 
1656  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1657  return DAG.getBlockAddress(BA, VT);
1658 
1659  if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1660  return getValue(Equiv->getGlobalValue());
1661 
1662  if (const auto *NC = dyn_cast<NoCFIValue>(C))
1663  return getValue(NC->getGlobalValue());
1664 
1665  VectorType *VecTy = cast<VectorType>(V->getType());
1666 
1667  // Now that we know the number and type of the elements, get that number of
1668  // elements into the Ops array based on what kind of constant it is.
1669  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1671  unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1672  for (unsigned i = 0; i != NumElements; ++i)
1673  Ops.push_back(getValue(CV->getOperand(i)));
1674 
1675  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1676  }
1677 
1678  if (isa<ConstantAggregateZero>(C)) {
1679  EVT EltVT =
1680  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1681 
1682  SDValue Op;
1683  if (EltVT.isFloatingPoint())
1684  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1685  else
1686  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1687 
1688  return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1689  }
1690 
1691  llvm_unreachable("Unknown vector constant");
1692  }
1693 
1694  // If this is a static alloca, generate it as the frameindex instead of
1695  // computation.
1696  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1698  FuncInfo.StaticAllocaMap.find(AI);
1699  if (SI != FuncInfo.StaticAllocaMap.end())
1700  return DAG.getFrameIndex(
1701  SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1702  }
1703 
1704  // If this is an instruction which fast-isel has deferred, select it now.
1705  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1706  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1707 
1708  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1709  Inst->getType(), std::nullopt);
1710  SDValue Chain = DAG.getEntryNode();
1711  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1712  }
1713 
1714  if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1715  return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1716 
1717  if (const auto *BB = dyn_cast<BasicBlock>(V))
1718  return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1719 
1720  llvm_unreachable("Can't get register for value!");
1721 }
1722 
1723 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1725  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1726  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1727  bool IsSEH = isAsynchronousEHPersonality(Pers);
1728  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1729  if (!IsSEH)
1730  CatchPadMBB->setIsEHScopeEntry();
1731  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1732  if (IsMSVCCXX || IsCoreCLR)
1733  CatchPadMBB->setIsEHFuncletEntry();
1734 }
1735 
1736 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1737  // Update machine-CFG edge.
1738  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1739  FuncInfo.MBB->addSuccessor(TargetMBB);
1740  TargetMBB->setIsEHCatchretTarget(true);
1742 
1744  bool IsSEH = isAsynchronousEHPersonality(Pers);
1745  if (IsSEH) {
1746  // If this is not a fall-through branch or optimizations are switched off,
1747  // emit the branch.
1748  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1749  TM.getOptLevel() == CodeGenOpt::None)
1751  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1752  return;
1753  }
1754 
1755  // Figure out the funclet membership for the catchret's successor.
1756  // This will be used by the FuncletLayout pass to determine how to order the
1757  // BB's.
1758  // A 'catchret' returns to the outer scope's color.
1759  Value *ParentPad = I.getCatchSwitchParentPad();
1760  const BasicBlock *SuccessorColor;
1761  if (isa<ConstantTokenNone>(ParentPad))
1762  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1763  else
1764  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1765  assert(SuccessorColor && "No parent funclet for catchret!");
1766  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1767  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1768 
1769  // Create the terminator node.
1771  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1772  DAG.getBasicBlock(SuccessorColorMBB));
1773  DAG.setRoot(Ret);
1774 }
1775 
1776 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1777  // Don't emit any special code for the cleanuppad instruction. It just marks
1778  // the start of an EH scope/funclet.
1781  if (Pers != EHPersonality::Wasm_CXX) {
1784  }
1785 }
1786 
1787 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1788 // not match, it is OK to add only the first unwind destination catchpad to the
1789 // successors, because there will be at least one invoke instruction within the
1790 // catch scope that points to the next unwind destination, if one exists, so
1791 // CFGSort cannot mess up with BB sorting order.
1792 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1793 // call within them, and catchpads only consisting of 'catch (...)' have a
1794 // '__cxa_end_catch' call within them, both of which generate invokes in case
1795 // the next unwind destination exists, i.e., the next unwind destination is not
1796 // the caller.)
1797 //
1798 // Having at most one EH pad successor is also simpler and helps later
1799 // transformations.
1800 //
1801 // For example,
1802 // current:
1803 // invoke void @foo to ... unwind label %catch.dispatch
1804 // catch.dispatch:
1805 // %0 = catchswitch within ... [label %catch.start] unwind label %next
1806 // catch.start:
1807 // ...
1808 // ... in this BB or some other child BB dominated by this BB there will be an
1809 // invoke that points to 'next' BB as an unwind destination
1810 //
1811 // next: ; We don't need to add this to 'current' BB's successor
1812 // ...
1814  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1815  BranchProbability Prob,
1816  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1817  &UnwindDests) {
1818  while (EHPadBB) {
1819  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1820  if (isa<CleanupPadInst>(Pad)) {
1821  // Stop on cleanup pads.
1822  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1823  UnwindDests.back().first->setIsEHScopeEntry();
1824  break;
1825  } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1826  // Add the catchpad handlers to the possible destinations. We don't
1827  // continue to the unwind destination of the catchswitch for wasm.
1828  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1829  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1830  UnwindDests.back().first->setIsEHScopeEntry();
1831  }
1832  break;
1833  } else {
1834  continue;
1835  }
1836  }
1837 }
1838 
1839 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1840 /// many places it could ultimately go. In the IR, we have a single unwind
1841 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1842 /// This function skips over imaginary basic blocks that hold catchswitch
1843 /// instructions, and finds all the "real" machine
1844 /// basic block destinations. As those destinations may not be successors of
1845 /// EHPadBB, here we also calculate the edge probability to those destinations.
1846 /// The passed-in Prob is the edge probability to EHPadBB.
1848  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1849  BranchProbability Prob,
1850  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1851  &UnwindDests) {
1852  EHPersonality Personality =
1854  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1855  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1856  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1857  bool IsSEH = isAsynchronousEHPersonality(Personality);
1858 
1859  if (IsWasmCXX) {
1860  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1861  assert(UnwindDests.size() <= 1 &&
1862  "There should be at most one unwind destination for wasm");
1863  return;
1864  }
1865 
1866  while (EHPadBB) {
1867  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1868  BasicBlock *NewEHPadBB = nullptr;
1869  if (isa<LandingPadInst>(Pad)) {
1870  // Stop on landingpads. They are not funclets.
1871  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1872  break;
1873  } else if (isa<CleanupPadInst>(Pad)) {
1874  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1875  // personalities.
1876  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1877  UnwindDests.back().first->setIsEHScopeEntry();
1878  UnwindDests.back().first->setIsEHFuncletEntry();
1879  break;
1880  } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1881  // Add the catchpad handlers to the possible destinations.
1882  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1883  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1884  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1885  if (IsMSVCCXX || IsCoreCLR)
1886  UnwindDests.back().first->setIsEHFuncletEntry();
1887  if (!IsSEH)
1888  UnwindDests.back().first->setIsEHScopeEntry();
1889  }
1890  NewEHPadBB = CatchSwitch->getUnwindDest();
1891  } else {
1892  continue;
1893  }
1894 
1895  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1896  if (BPI && NewEHPadBB)
1897  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1898  EHPadBB = NewEHPadBB;
1899  }
1900 }
1901 
1902 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1903  // Update successor info.
1905  auto UnwindDest = I.getUnwindDest();
1907  BranchProbability UnwindDestProb =
1908  (BPI && UnwindDest)
1909  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1911  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1912  for (auto &UnwindDest : UnwindDests) {
1913  UnwindDest.first->setIsEHPad();
1914  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1915  }
1917 
1918  // Create the terminator node.
1919  SDValue Ret =
1921  DAG.setRoot(Ret);
1922 }
1923 
1924 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1925  report_fatal_error("visitCatchSwitch not yet implemented!");
1926 }
1927 
1928 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1929  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1930  auto &DL = DAG.getDataLayout();
1931  SDValue Chain = getControlRoot();
1933  SmallVector<SDValue, 8> OutVals;
1934 
1935  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1936  // lower
1937  //
1938  // %val = call <ty> @llvm.experimental.deoptimize()
1939  // ret <ty> %val
1940  //
1941  // differently.
1942  if (I.getParent()->getTerminatingDeoptimizeCall()) {
1944  return;
1945  }
1946 
1947  if (!FuncInfo.CanLowerReturn) {
1948  unsigned DemoteReg = FuncInfo.DemoteRegister;
1949  const Function *F = I.getParent()->getParent();
1950 
1951  // Emit a store of the return value through the virtual register.
1952  // Leave Outs empty so that LowerReturn won't try to load return
1953  // registers the usual way.
1954  SmallVector<EVT, 1> PtrValueVTs;
1955  ComputeValueVTs(TLI, DL,
1956  F->getReturnType()->getPointerTo(
1958  PtrValueVTs);
1959 
1960  SDValue RetPtr =
1961  DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
1962  SDValue RetOp = getValue(I.getOperand(0));
1963 
1964  SmallVector<EVT, 4> ValueVTs, MemVTs;
1966  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1967  &Offsets);
1968  unsigned NumValues = ValueVTs.size();
1969 
1970  SmallVector<SDValue, 4> Chains(NumValues);
1971  Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1972  for (unsigned i = 0; i != NumValues; ++i) {
1973  // An aggregate return value cannot wrap around the address space, so
1974  // offsets to its parts don't wrap either.
1977 
1978  SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1979  if (MemVTs[i] != ValueVTs[i])
1980  Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1981  Chains[i] = DAG.getStore(
1982  Chain, getCurSDLoc(), Val,
1983  // FIXME: better loc info would be nice.
1985  commonAlignment(BaseAlign, Offsets[i]));
1986  }
1987 
1989  MVT::Other, Chains);
1990  } else if (I.getNumOperands() != 0) {
1991  SmallVector<EVT, 4> ValueVTs;
1992  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1993  unsigned NumValues = ValueVTs.size();
1994  if (NumValues) {
1995  SDValue RetOp = getValue(I.getOperand(0));
1996 
1997  const Function *F = I.getParent()->getParent();
1998 
1999  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2000  I.getOperand(0)->getType(), F->getCallingConv(),
2001  /*IsVarArg*/ false, DL);
2002 
2003  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2004  if (F->getAttributes().hasRetAttr(Attribute::SExt))
2005  ExtendKind = ISD::SIGN_EXTEND;
2006  else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2007  ExtendKind = ISD::ZERO_EXTEND;
2008 
2009  LLVMContext &Context = F->getContext();
2010  bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2011 
2012  for (unsigned j = 0; j != NumValues; ++j) {
2013  EVT VT = ValueVTs[j];
2014 
2015  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2016  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2017 
2018  CallingConv::ID CC = F->getCallingConv();
2019 
2020  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2021  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2022  SmallVector<SDValue, 4> Parts(NumParts);
2024  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2025  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2026 
2027  // 'inreg' on function refers to return value
2029  if (RetInReg)
2030  Flags.setInReg();
2031 
2032  if (I.getOperand(0)->getType()->isPointerTy()) {
2033  Flags.setPointer();
2034  Flags.setPointerAddrSpace(
2035  cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2036  }
2037 
2038  if (NeedsRegBlock) {
2039  Flags.setInConsecutiveRegs();
2040  if (j == NumValues - 1)
2041  Flags.setInConsecutiveRegsLast();
2042  }
2043 
2044  // Propagate extension type if any
2045  if (ExtendKind == ISD::SIGN_EXTEND)
2046  Flags.setSExt();
2047  else if (ExtendKind == ISD::ZERO_EXTEND)
2048  Flags.setZExt();
2049 
2050  for (unsigned i = 0; i < NumParts; ++i) {
2051  Outs.push_back(ISD::OutputArg(Flags,
2052  Parts[i].getValueType().getSimpleVT(),
2053  VT, /*isfixed=*/true, 0, 0));
2054  OutVals.push_back(Parts[i]);
2055  }
2056  }
2057  }
2058  }
2059 
2060  // Push in swifterror virtual register as the last element of Outs. This makes
2061  // sure swifterror virtual register will be returned in the swifterror
2062  // physical register.
2063  const Function *F = I.getParent()->getParent();
2064  if (TLI.supportSwiftError() &&
2065  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2066  assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2068  Flags.setSwiftError();
2069  Outs.push_back(ISD::OutputArg(
2070  Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2071  /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2072  // Create SDNode for the swifterror virtual register.
2073  OutVals.push_back(
2076  EVT(TLI.getPointerTy(DL))));
2077  }
2078 
2079  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2080  CallingConv::ID CallConv =
2083  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2084 
2085  // Verify that the target's LowerReturn behaved as expected.
2086  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2087  "LowerReturn didn't return a valid chain!");
2088 
2089  // Update the DAG with the new chain value resulting from return lowering.
2090  DAG.setRoot(Chain);
2091 }
2092 
2093 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2094 /// created for it, emit nodes to copy the value into the virtual
2095 /// registers.
2097  // Skip empty types
2098  if (V->getType()->isEmptyTy())
2099  return;
2100 
2102  if (VMI != FuncInfo.ValueMap.end()) {
2103  assert(!V->use_empty() && "Unused value assigned virtual registers!");
2104  CopyValueToVirtualRegister(V, VMI->second);
2105  }
2106 }
2107 
2108 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2109 /// the current basic block, add it to ValueMap now so that we'll get a
2110 /// CopyTo/FromReg.
2112  // No need to export constants.
2113  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2114 
2115  // Already exported?
2116  if (FuncInfo.isExportedInst(V)) return;
2117 
2118  unsigned Reg = FuncInfo.InitializeRegForValue(V);
2120 }
2121 
2123  const BasicBlock *FromBB) {
2124  // The operands of the setcc have to be in this block. We don't know
2125  // how to export them from some other block.
2126  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2127  // Can export from current BB.
2128  if (VI->getParent() == FromBB)
2129  return true;
2130 
2131  // Is already exported, noop.
2132  return FuncInfo.isExportedInst(V);
2133  }
2134 
2135  // If this is an argument, we can export it if the BB is the entry block or
2136  // if it is already exported.
2137  if (isa<Argument>(V)) {
2138  if (FromBB->isEntryBlock())
2139  return true;
2140 
2141  // Otherwise, can only export this if it is already exported.
2142  return FuncInfo.isExportedInst(V);
2143  }
2144 
2145  // Otherwise, constants can always be exported.
2146  return true;
2147 }
2148 
2149 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2151 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2152  const MachineBasicBlock *Dst) const {
2154  const BasicBlock *SrcBB = Src->getBasicBlock();
2155  const BasicBlock *DstBB = Dst->getBasicBlock();
2156  if (!BPI) {
2157  // If BPI is not available, set the default probability as 1 / N, where N is
2158  // the number of successors.
2159  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2160  return BranchProbability(1, SuccSize);
2161  }
2162  return BPI->getEdgeProbability(SrcBB, DstBB);
2163 }
2164 
2165 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2166  MachineBasicBlock *Dst,
2167  BranchProbability Prob) {
2168  if (!FuncInfo.BPI)
2169  Src->addSuccessorWithoutProb(Dst);
2170  else {
2171  if (Prob.isUnknown())
2172  Prob = getEdgeProbability(Src, Dst);
2173  Src->addSuccessor(Dst, Prob);
2174  }
2175 }
2176 
2177 static bool InBlock(const Value *V, const BasicBlock *BB) {
2178  if (const Instruction *I = dyn_cast<Instruction>(V))
2179  return I->getParent() == BB;
2180  return true;
2181 }
2182 
2183 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2184 /// This function emits a branch and is used at the leaves of an OR or an
2185 /// AND operator tree.
2186 void
2189  MachineBasicBlock *FBB,
2190  MachineBasicBlock *CurBB,
2191  MachineBasicBlock *SwitchBB,
2192  BranchProbability TProb,
2193  BranchProbability FProb,
2194  bool InvertCond) {
2195  const BasicBlock *BB = CurBB->getBasicBlock();
2196 
2197  // If the leaf of the tree is a comparison, merge the condition into
2198  // the caseblock.
2199  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2200  // The operands of the cmp have to be in this block. We don't know
2201  // how to export them from some other block. If this is the first block
2202  // of the sequence, no exporting is needed.
2203  if (CurBB == SwitchBB ||
2204  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2205  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2206  ISD::CondCode Condition;
2207  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2208  ICmpInst::Predicate Pred =
2209  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2210  Condition = getICmpCondCode(Pred);
2211  } else {
2212  const FCmpInst *FC = cast<FCmpInst>(Cond);
2213  FCmpInst::Predicate Pred =
2214  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2215  Condition = getFCmpCondCode(Pred);
2216  if (TM.Options.NoNaNsFPMath)
2217  Condition = getFCmpCodeWithoutNaN(Condition);
2218  }
2219 
2220  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2221  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2222  SL->SwitchCases.push_back(CB);
2223  return;
2224  }
2225  }
2226 
2227  // Create a CaseBlock record representing this branch.
2228  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2229  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2230  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2231  SL->SwitchCases.push_back(CB);
2232 }
2233 
2236  MachineBasicBlock *FBB,
2237  MachineBasicBlock *CurBB,
2238  MachineBasicBlock *SwitchBB,
2240  BranchProbability TProb,
2241  BranchProbability FProb,
2242  bool InvertCond) {
2243  // Skip over not part of the tree and remember to invert op and operands at
2244  // next level.
2245  Value *NotCond;
2246  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2247  InBlock(NotCond, CurBB->getBasicBlock())) {
2248  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2249  !InvertCond);
2250  return;
2251  }
2252 
2253  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2254  const Value *BOpOp0, *BOpOp1;
2255  // Compute the effective opcode for Cond, taking into account whether it needs
2256  // to be inverted, e.g.
2257  // and (not (or A, B)), C
2258  // gets lowered as
2259  // and (and (not A, not B), C)
2261  if (BOp) {
2262  BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2263  ? Instruction::And
2264  : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2265  ? Instruction::Or
2266  : (Instruction::BinaryOps)0);
2267  if (InvertCond) {
2268  if (BOpc == Instruction::And)
2269  BOpc = Instruction::Or;
2270  else if (BOpc == Instruction::Or)
2271  BOpc = Instruction::And;
2272  }
2273  }
2274 
2275  // If this node is not part of the or/and tree, emit it as a branch.
2276  // Note that all nodes in the tree should have same opcode.
2277  bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2278  if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2279  !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2280  !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2281  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2282  TProb, FProb, InvertCond);
2283  return;
2284  }
2285 
2286  // Create TmpBB after CurBB.
2287  MachineFunction::iterator BBI(CurBB);
2290  CurBB->getParent()->insert(++BBI, TmpBB);
2291 
2292  if (Opc == Instruction::Or) {
2293  // Codegen X | Y as:
2294  // BB1:
2295  // jmp_if_X TBB
2296  // jmp TmpBB
2297  // TmpBB:
2298  // jmp_if_Y TBB
2299  // jmp FBB
2300  //
2301 
2302  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2303  // The requirement is that
2304  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2305  // = TrueProb for original BB.
2306  // Assuming the original probabilities are A and B, one choice is to set
2307  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2308  // A/(1+B) and 2B/(1+B). This choice assumes that
2309  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2310  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2311  // TmpBB, but the math is more complicated.
2312 
2313  auto NewTrueProb = TProb / 2;
2314  auto NewFalseProb = TProb / 2 + FProb;
2315  // Emit the LHS condition.
2316  FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2317  NewFalseProb, InvertCond);
2318 
2319  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2320  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2321  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2322  // Emit the RHS condition into TmpBB.
2323  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2324  Probs[1], InvertCond);
2325  } else {
2326  assert(Opc == Instruction::And && "Unknown merge op!");
2327  // Codegen X & Y as:
2328  // BB1:
2329  // jmp_if_X TmpBB
2330  // jmp FBB
2331  // TmpBB:
2332  // jmp_if_Y TBB
2333  // jmp FBB
2334  //
2335  // This requires creation of TmpBB after CurBB.
2336 
2337  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2338  // The requirement is that
2339  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2340  // = FalseProb for original BB.
2341  // Assuming the original probabilities are A and B, one choice is to set
2342  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2343  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2344  // TrueProb for BB1 * FalseProb for TmpBB.
2345 
2346  auto NewTrueProb = TProb + FProb / 2;
2347  auto NewFalseProb = FProb / 2;
2348  // Emit the LHS condition.
2349  FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2350  NewFalseProb, InvertCond);
2351 
2352  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2353  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2354  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2355  // Emit the RHS condition into TmpBB.
2356  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2357  Probs[1], InvertCond);
2358  }
2359 }
2360 
2361 /// If the set of cases should be emitted as a series of branches, return true.
2362 /// If we should emit this as a bunch of and/or'd together conditions, return
2363 /// false.
2364 bool
2365 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2366  if (Cases.size() != 2) return true;
2367 
2368  // If this is two comparisons of the same values or'd or and'd together, they
2369  // will get folded into a single comparison, so don't emit two blocks.
2370  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2371  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2372  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2373  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2374  return false;
2375  }
2376 
2377  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2378  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2379  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2380  Cases[0].CC == Cases[1].CC &&
2381  isa<Constant>(Cases[0].CmpRHS) &&
2382  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2383  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2384  return false;
2385  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2386  return false;
2387  }
2388 
2389  return true;
2390 }
2391 
2392 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2393  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2394 
2395  // Update machine-CFG edges.
2396  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2397 
2398  if (I.isUnconditional()) {
2399  // Update machine-CFG edges.
2400  BrMBB->addSuccessor(Succ0MBB);
2401 
2402  // If this is not a fall-through branch or optimizations are switched off,
2403  // emit the branch.
2404  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2407  DAG.getBasicBlock(Succ0MBB)));
2408 
2409  return;
2410  }
2411 
2412  // If this condition is one of the special cases we handle, do special stuff
2413  // now.
2414  const Value *CondVal = I.getCondition();
2415  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2416 
2417  // If this is a series of conditions that are or'd or and'd together, emit
2418  // this as a sequence of branches instead of setcc's with and/or operations.
2419  // As long as jumps are not expensive (exceptions for multi-use logic ops,
2420  // unpredictable branches, and vector extracts because those jumps are likely
2421  // expensive for any target), this should improve performance.
2422  // For example, instead of something like:
2423  // cmp A, B
2424  // C = seteq
2425  // cmp D, E
2426  // F = setle
2427  // or C, F
2428  // jnz foo
2429  // Emit:
2430  // cmp A, B
2431  // je foo
2432  // cmp D, E
2433  // jle foo
2434  const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2435  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2436  BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2437  Value *Vec;
2438  const Value *BOp0, *BOp1;
2440  if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2441  Opcode = Instruction::And;
2442  else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2443  Opcode = Instruction::Or;
2444 
2445  if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2446  match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2447  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2448  getEdgeProbability(BrMBB, Succ0MBB),
2449  getEdgeProbability(BrMBB, Succ1MBB),
2450  /*InvertCond=*/false);
2451  // If the compares in later blocks need to use values not currently
2452  // exported from this block, export them now. This block should always
2453  // be the first entry.
2454  assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2455 
2456  // Allow some cases to be rejected.
2457  if (ShouldEmitAsBranches(SL->SwitchCases)) {
2458  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2459  ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2460  ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2461  }
2462 
2463  // Emit the branch for this block.
2464  visitSwitchCase(SL->SwitchCases[0], BrMBB);
2465  SL->SwitchCases.erase(SL->SwitchCases.begin());
2466  return;
2467  }
2468 
2469  // Okay, we decided not to do this, remove any inserted MBB's and clear
2470  // SwitchCases.
2471  for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2472  FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2473 
2474  SL->SwitchCases.clear();
2475  }
2476  }
2477 
2478  // Create a CaseBlock record representing this branch.
2479  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2480  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2481 
2482  // Use visitSwitchCase to actually insert the fast branch sequence for this
2483  // cond branch.
2484  visitSwitchCase(CB, BrMBB);
2485 }
2486 
2487 /// visitSwitchCase - Emits the necessary code to represent a single node in
2488 /// the binary search tree resulting from lowering a switch instruction.
2490  MachineBasicBlock *SwitchBB) {
2491  SDValue Cond;
2492  SDValue CondLHS = getValue(CB.CmpLHS);
2493  SDLoc dl = CB.DL;
2494 
2495  if (CB.CC == ISD::SETTRUE) {
2496  // Branch or fall through to TrueBB.
2497  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2498  SwitchBB->normalizeSuccProbs();
2499  if (CB.TrueBB != NextBlock(SwitchBB)) {
2501  DAG.getBasicBlock(CB.TrueBB)));
2502  }
2503  return;
2504  }
2505 
2506  auto &TLI = DAG.getTargetLoweringInfo();
2507  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2508 
2509  // Build the setcc now.
2510  if (!CB.CmpMHS) {
2511  // Fold "(X == true)" to X and "(X == false)" to !X to
2512  // handle common cases produced by branch lowering.
2513  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2514  CB.CC == ISD::SETEQ)
2515  Cond = CondLHS;
2516  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2517  CB.CC == ISD::SETEQ) {
2518  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2519  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2520  } else {
2521  SDValue CondRHS = getValue(CB.CmpRHS);
2522 
2523  // If a pointer's DAG type is larger than its memory type then the DAG
2524  // values are zero-extended. This breaks signed comparisons so truncate
2525  // back to the underlying type before doing the compare.
2526  if (CondLHS.getValueType() != MemVT) {
2527  CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2528  CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2529  }
2530  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2531  }
2532  } else {
2533  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2534 
2535  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2536  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2537 
2538  SDValue CmpOp = getValue(CB.CmpMHS);
2539  EVT VT = CmpOp.getValueType();
2540 
2541  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2542  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2543  ISD::SETLE);
2544  } else {
2545  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2546  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2547  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2548  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2549  }
2550  }
2551 
2552  // Update successor info
2553  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2554  // TrueBB and FalseBB are always different unless the incoming IR is
2555  // degenerate. This only happens when running llc on weird IR.
2556  if (CB.TrueBB != CB.FalseBB)
2557  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2558  SwitchBB->normalizeSuccProbs();
2559 
2560  // If the lhs block is the next block, invert the condition so that we can
2561  // fall through to the lhs instead of the rhs block.
2562  if (CB.TrueBB == NextBlock(SwitchBB)) {
2563  std::swap(CB.TrueBB, CB.FalseBB);
2564  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2565  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2566  }
2567 
2568  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2570  DAG.getBasicBlock(CB.TrueBB));
2571 
2572  setValue(CurInst, BrCond);
2573 
2574  // Insert the false branch. Do this even if it's a fall through branch,
2575  // this makes it easier to do DAG optimizations which require inverting
2576  // the branch condition.
2577  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2578  DAG.getBasicBlock(CB.FalseBB));
2579 
2580  DAG.setRoot(BrCond);
2581 }
2582 
2583 /// visitJumpTable - Emit JumpTable node in the current MBB
2585  // Emit the code for the jump table
2586  assert(JT.Reg != -1U && "Should lower JT Header first!");
2589  JT.Reg, PTy);
2590  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2591  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2592  MVT::Other, Index.getValue(1),
2593  Table, Index);
2594  DAG.setRoot(BrJumpTable);
2595 }
2596 
2597 /// visitJumpTableHeader - This function emits necessary code to produce index
2598 /// in the JumpTable from switch case.
2600  JumpTableHeader &JTH,
2601  MachineBasicBlock *SwitchBB) {
2602  SDLoc dl = getCurSDLoc();
2603 
2604  // Subtract the lowest switch case value from the value being switched on.
2605  SDValue SwitchOp = getValue(JTH.SValue);
2606  EVT VT = SwitchOp.getValueType();
2607  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2608  DAG.getConstant(JTH.First, dl, VT));
2609 
2610  // The SDNode we just created, which holds the value being switched on minus
2611  // the smallest case value, needs to be copied to a virtual register so it
2612  // can be used as an index into the jump table in a subsequent basic block.
2613  // This value may be smaller or larger than the target's pointer type, and
2614  // therefore require extension or truncating.
2615  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2616  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2617 
2618  unsigned JumpTableReg =
2620  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2621  JumpTableReg, SwitchOp);
2622  JT.Reg = JumpTableReg;
2623 
2624  if (!JTH.FallthroughUnreachable) {
2625  // Emit the range check for the jump table, and branch to the default block
2626  // for the switch statement if the value being switched on exceeds the
2627  // largest case in the switch.
2628  SDValue CMP = DAG.getSetCC(
2630  Sub.getValueType()),
2631  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2632 
2633  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2634  MVT::Other, CopyTo, CMP,
2635  DAG.getBasicBlock(JT.Default));
2636 
2637  // Avoid emitting unnecessary branches to the next block.
2638  if (JT.MBB != NextBlock(SwitchBB))
2639  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2640  DAG.getBasicBlock(JT.MBB));
2641 
2642  DAG.setRoot(BrCond);
2643  } else {
2644  // Avoid emitting unnecessary branches to the next block.
2645  if (JT.MBB != NextBlock(SwitchBB))
2646  DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2647  DAG.getBasicBlock(JT.MBB)));
2648  else
2649  DAG.setRoot(CopyTo);
2650  }
2651 }
2652 
2653 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2654 /// variable if there exists one.
2656  SDValue &Chain) {
2657  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2658  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2659  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2660  MachineFunction &MF = DAG.getMachineFunction();
2661  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2662  MachineSDNode *Node =
2663  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2664  if (Global) {
2665  MachinePointerInfo MPInfo(Global);
2669  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2670  DAG.setNodeMemRefs(Node, {MemRef});
2671  }
2672  if (PtrTy != PtrMemTy)
2673  return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2674  return SDValue(Node, 0);
2675 }
2676 
2677 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2678 /// tail spliced into a stack protector check success bb.
2679 ///
2680 /// For a high level explanation of how this fits into the stack protector
2681 /// generation see the comment on the declaration of class
2682 /// StackProtectorDescriptor.
2684  MachineBasicBlock *ParentBB) {
2685 
2686  // First create the loads to the guard/stack slot for the comparison.
2687  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2688  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2689  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2690 
2691  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2692  int FI = MFI.getStackProtectorIndex();
2693 
2694  SDValue Guard;
2695  SDLoc dl = getCurSDLoc();
2696  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2697  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2698  Align Align =
2700 
2701  // Generate code to load the content of the guard slot.
2702  SDValue GuardVal = DAG.getLoad(
2703  PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2706 
2707  if (TLI.useStackGuardXorFP())
2708  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2709 
2710  // Retrieve guard check function, nullptr if instrumentation is inlined.
2711  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2712  // The target provides a guard check function to validate the guard value.
2713  // Generate a call to that function with the content of the guard slot as
2714  // argument.
2715  FunctionType *FnTy = GuardCheckFn->getFunctionType();
2716  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2717 
2720  Entry.Node = GuardVal;
2721  Entry.Ty = FnTy->getParamType(0);
2722  if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2723  Entry.IsInReg = true;
2724  Args.push_back(Entry);
2725 
2727  CLI.setDebugLoc(getCurSDLoc())
2729  .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2730  getValue(GuardCheckFn), std::move(Args));
2731 
2732  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2733  DAG.setRoot(Result.second);
2734  return;
2735  }
2736 
2737  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2738  // Otherwise, emit a volatile load to retrieve the stack guard value.
2739  SDValue Chain = DAG.getEntryNode();
2740  if (TLI.useLoadStackGuardNode()) {
2741  Guard = getLoadStackGuard(DAG, dl, Chain);
2742  } else {
2743  const Value *IRGuard = TLI.getSDagStackGuard(M);
2744  SDValue GuardPtr = getValue(IRGuard);
2745 
2746  Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2747  MachinePointerInfo(IRGuard, 0), Align,
2749  }
2750 
2751  // Perform the comparison via a getsetcc.
2753  *DAG.getContext(),
2754  Guard.getValueType()),
2755  Guard, GuardVal, ISD::SETNE);
2756 
2757  // If the guard/stackslot do not equal, branch to failure MBB.
2758  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2759  MVT::Other, GuardVal.getOperand(0),
2760  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2761  // Otherwise branch to success MBB.
2762  SDValue Br = DAG.getNode(ISD::BR, dl,
2763  MVT::Other, BrCond,
2765 
2766  DAG.setRoot(Br);
2767 }
2768 
2769 /// Codegen the failure basic block for a stack protector check.
2770 ///
2771 /// A failure stack protector machine basic block consists simply of a call to
2772 /// __stack_chk_fail().
2773 ///
2774 /// For a high level explanation of how this fits into the stack protector
2775 /// generation see the comment on the declaration of class
2776 /// StackProtectorDescriptor.
2777 void
2779  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2781  CallOptions.setDiscardResult(true);
2782  SDValue Chain =
2783  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2784  std::nullopt, CallOptions, getCurSDLoc())
2785  .second;
2786  // On PS4/PS5, the "return address" must still be within the calling
2787  // function, even if it's at the very end, so emit an explicit TRAP here.
2788  // Passing 'true' for doesNotReturn above won't generate the trap for us.
2789  if (TM.getTargetTriple().isPS())
2790  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2791  // WebAssembly needs an unreachable instruction after a non-returning call,
2792  // because the function return type can be different from __stack_chk_fail's
2793  // return type (void).
2794  if (TM.getTargetTriple().isWasm())
2795  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2796 
2797  DAG.setRoot(Chain);
2798 }
2799 
2800 /// visitBitTestHeader - This function emits necessary code to produce value
2801 /// suitable for "bit tests"
2803  MachineBasicBlock *SwitchBB) {
2804  SDLoc dl = getCurSDLoc();
2805 
2806  // Subtract the minimum value.
2807  SDValue SwitchOp = getValue(B.SValue);
2808  EVT VT = SwitchOp.getValueType();
2809  SDValue RangeSub =
2810  DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2811 
2812  // Determine the type of the test operands.
2813  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2814  bool UsePtrType = false;
2815  if (!TLI.isTypeLegal(VT)) {
2816  UsePtrType = true;
2817  } else {
2818  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2819  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2820  // Switch table case range are encoded into series of masks.
2821  // Just use pointer type, it's guaranteed to fit.
2822  UsePtrType = true;
2823  break;
2824  }
2825  }
2826  SDValue Sub = RangeSub;
2827  if (UsePtrType) {
2828  VT = TLI.getPointerTy(DAG.getDataLayout());
2829  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2830  }
2831 
2832  B.RegVT = VT.getSimpleVT();
2833  B.Reg = FuncInfo.CreateReg(B.RegVT);
2834  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2835 
2836  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2837 
2838  if (!B.FallthroughUnreachable)
2839  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2840  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2841  SwitchBB->normalizeSuccProbs();
2842 
2843  SDValue Root = CopyTo;
2844  if (!B.FallthroughUnreachable) {
2845  // Conditional branch to the default block.
2846  SDValue RangeCmp = DAG.getSetCC(dl,
2848  RangeSub.getValueType()),
2849  RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2850  ISD::SETUGT);
2851 
2852  Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2853  DAG.getBasicBlock(B.Default));
2854  }
2855 
2856  // Avoid emitting unnecessary branches to the next block.
2857  if (MBB != NextBlock(SwitchBB))
2858  Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2859 
2860  DAG.setRoot(Root);
2861 }
2862 
2863 /// visitBitTestCase - this function produces one "bit test"
2865  MachineBasicBlock* NextMBB,
2866  BranchProbability BranchProbToNext,
2867  unsigned Reg,
2868  BitTestCase &B,
2869  MachineBasicBlock *SwitchBB) {
2870  SDLoc dl = getCurSDLoc();
2871  MVT VT = BB.RegVT;
2872  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2873  SDValue Cmp;
2874  unsigned PopCount = countPopulation(B.Mask);
2875  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2876  if (PopCount == 1) {
2877  // Testing for a single bit; just compare the shift count with what it
2878  // would need to be to shift a 1 bit in that position.
2879  Cmp = DAG.getSetCC(
2880  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2881  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2882  ISD::SETEQ);
2883  } else if (PopCount == BB.Range) {
2884  // There is only one zero bit in the range, test for it directly.
2885  Cmp = DAG.getSetCC(
2886  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2887  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2888  ISD::SETNE);
2889  } else {
2890  // Make desired shift
2891  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2892  DAG.getConstant(1, dl, VT), ShiftOp);
2893 
2894  // Emit bit tests and jumps
2895  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2896  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2897  Cmp = DAG.getSetCC(
2898  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2899  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2900  }
2901 
2902  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2903  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2904  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2905  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2906  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2907  // one as they are relative probabilities (and thus work more like weights),
2908  // and hence we need to normalize them to let the sum of them become one.
2909  SwitchBB->normalizeSuccProbs();
2910 
2911  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2913  Cmp, DAG.getBasicBlock(B.TargetBB));
2914 
2915  // Avoid emitting unnecessary branches to the next block.
2916  if (NextMBB != NextBlock(SwitchBB))
2917  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2918  DAG.getBasicBlock(NextMBB));
2919 
2920  DAG.setRoot(BrAnd);
2921 }
2922 
2923 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2924  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2925 
2926  // Retrieve successors. Look through artificial IR level blocks like
2927  // catchswitch for successors.
2928  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2929  const BasicBlock *EHPadBB = I.getSuccessor(1);
2930 
2931  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2932  // have to do anything here to lower funclet bundles.
2933  assert(!I.hasOperandBundlesOtherThan(
2934  {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2935  LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2936  LLVMContext::OB_cfguardtarget,
2937  LLVMContext::OB_clang_arc_attachedcall}) &&
2938  "Cannot lower invokes with arbitrary operand bundles yet!");
2939 
2940  const Value *Callee(I.getCalledOperand());
2941  const Function *Fn = dyn_cast<Function>(Callee);
2942  if (isa<InlineAsm>(Callee))
2943  visitInlineAsm(I, EHPadBB);
2944  else if (Fn && Fn->isIntrinsic()) {
2945  switch (Fn->getIntrinsicID()) {
2946  default:
2947  llvm_unreachable("Cannot invoke this intrinsic");
2948  case Intrinsic::donothing:
2949  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2950  case Intrinsic::seh_try_begin:
2951  case Intrinsic::seh_scope_begin:
2952  case Intrinsic::seh_try_end:
2953  case Intrinsic::seh_scope_end:
2954  break;
2955  case Intrinsic::experimental_patchpoint_void:
2956  case Intrinsic::experimental_patchpoint_i64:
2957  visitPatchpoint(I, EHPadBB);
2958  break;
2959  case Intrinsic::experimental_gc_statepoint:
2960  LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2961  break;
2962  case Intrinsic::wasm_rethrow: {
2963  // This is usually done in visitTargetIntrinsic, but this intrinsic is
2964  // special because it can be invoked, so we manually lower it to a DAG
2965  // node here.
2967  Ops.push_back(getRoot()); // inchain
2968  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2969  Ops.push_back(
2970  DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2971  TLI.getPointerTy(DAG.getDataLayout())));
2972  SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2974  break;
2975  }
2976  }
2977  } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2978  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2979  // Eventually we will support lowering the @llvm.experimental.deoptimize
2980  // intrinsic, and right now there are no plans to support other intrinsics
2981  // with deopt state.
2982  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2983  } else {
2984  LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2985  }
2986 
2987  // If the value of the invoke is used outside of its defining block, make it
2988  // available as a virtual register.
2989  // We already took care of the exported value for the statepoint instruction
2990  // during call to the LowerStatepoint.
2991  if (!isa<GCStatepointInst>(I)) {
2993  }
2994 
2997  BranchProbability EHPadBBProb =
2998  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3000  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3001 
3002  // Update successor info.
3003  addSuccessorWithProb(InvokeMBB, Return);
3004  for (auto &UnwindDest : UnwindDests) {
3005  UnwindDest.first->setIsEHPad();
3006  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3007  }
3008  InvokeMBB->normalizeSuccProbs();
3009 
3010  // Drop into normal successor.
3012  DAG.getBasicBlock(Return)));
3013 }
3014 
3015 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3016  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3017 
3018  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3019  // have to do anything here to lower funclet bundles.
3020  assert(!I.hasOperandBundlesOtherThan(
3021  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3022  "Cannot lower callbrs with arbitrary operand bundles yet!");
3023 
3024  assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3025  visitInlineAsm(I);
3027 
3028  // Retrieve successors.
3030  Dests.insert(I.getDefaultDest());
3031  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3032 
3033  // Update successor info.
3034  addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3035  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3036  BasicBlock *Dest = I.getIndirectDest(i);
3038  Target->setIsInlineAsmBrIndirectTarget();
3039  Target->setMachineBlockAddressTaken();
3040  Target->setLabelMustBeEmitted();
3041  // Don't add duplicate machine successors.
3042  if (Dests.insert(Dest).second)
3043  addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3044  }
3045  CallBrMBB->normalizeSuccProbs();
3046 
3047  // Drop into default successor.
3050  DAG.getBasicBlock(Return)));
3051 }
3052 
3053 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3054  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3055 }
3056 
3057 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3058  assert(FuncInfo.MBB->isEHPad() &&
3059  "Call to landingpad not in landing pad!");
3060 
3061  // If there aren't registers to copy the values into (e.g., during SjLj
3062  // exceptions), then don't bother to create these DAG nodes.
3063  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3064  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3065  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3066  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3067  return;
3068 
3069  // If landingpad's return type is token type, we don't create DAG nodes
3070  // for its exception pointer and selector value. The extraction of exception
3071  // pointer or selector value from token type landingpads is not currently
3072  // supported.
3073  if (LP.getType()->isTokenTy())
3074  return;
3075 
3076  SmallVector<EVT, 2> ValueVTs;
3077  SDLoc dl = getCurSDLoc();
3078  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3079  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3080 
3081  // Get the two live-in registers as SDValues. The physregs have already been
3082  // copied into virtual registers.
3083  SDValue Ops[2];
3085  Ops[0] = DAG.getZExtOrTrunc(
3088  TLI.getPointerTy(DAG.getDataLayout())),
3089  dl, ValueVTs[0]);
3090  } else {
3091  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3092  }
3093  Ops[1] = DAG.getZExtOrTrunc(
3096  TLI.getPointerTy(DAG.getDataLayout())),
3097  dl, ValueVTs[1]);
3098 
3099  // Merge into one.
3101  DAG.getVTList(ValueVTs), Ops);
3102  setValue(&LP, Res);
3103 }
3104 
3106  MachineBasicBlock *Last) {
3107  // Update JTCases.
3108  for (JumpTableBlock &JTB : SL->JTCases)
3109  if (JTB.first.HeaderBB == First)
3110  JTB.first.HeaderBB = Last;
3111 
3112  // Update BitTestCases.
3113  for (BitTestBlock &BTB : SL->BitTestCases)
3114  if (BTB.Parent == First)
3115  BTB.Parent = Last;
3116 }
3117 
3118 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3119  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3120 
3121  // Update machine-CFG edges with unique successors.
3123  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3124  BasicBlock *BB = I.getSuccessor(i);
3125  bool Inserted = Done.insert(BB).second;
3126  if (!Inserted)
3127  continue;
3128 
3130  addSuccessorWithProb(IndirectBrMBB, Succ);
3131  }
3132  IndirectBrMBB->normalizeSuccProbs();
3133 
3136  getValue(I.getAddress())));
3137 }
3138 
3139 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3141  return;
3142 
3143  // We may be able to ignore unreachable behind a noreturn call.
3145  const BasicBlock &BB = *I.getParent();
3146  if (&I != &BB.front()) {
3148  std::prev(BasicBlock::const_iterator(&I));
3149  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3150  if (Call->doesNotReturn())
3151  return;
3152  }
3153  }
3154  }
3155 
3157 }
3158 
3159 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3161  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3162  Flags.copyFMF(*FPOp);
3163 
3164  SDValue Op = getValue(I.getOperand(0));
3165  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3166  Op, Flags);
3167  setValue(&I, UnNodeValue);
3168 }
3169 
3170 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3172  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3173  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3174  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3175  }
3176  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3177  Flags.setExact(ExactOp->isExact());
3178  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3179  Flags.copyFMF(*FPOp);
3180 
3181  SDValue Op1 = getValue(I.getOperand(0));
3182  SDValue Op2 = getValue(I.getOperand(1));
3183  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3184  Op1, Op2, Flags);
3185  setValue(&I, BinNodeValue);
3186 }
3187 
3188 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3189  SDValue Op1 = getValue(I.getOperand(0));
3190  SDValue Op2 = getValue(I.getOperand(1));
3191 
3193  Op1.getValueType(), DAG.getDataLayout());
3194 
3195  // Coerce the shift amount to the right type if we can. This exposes the
3196  // truncate or zext to optimization early.
3197  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3198  assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3199  "Unexpected shift type");
3200  Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3201  }
3202 
3203  bool nuw = false;
3204  bool nsw = false;
3205  bool exact = false;
3206 
3207  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3208 
3209  if (const OverflowingBinaryOperator *OFBinOp =
3210  dyn_cast<const OverflowingBinaryOperator>(&I)) {
3211  nuw = OFBinOp->hasNoUnsignedWrap();
3212  nsw = OFBinOp->hasNoSignedWrap();
3213  }
3214  if (const PossiblyExactOperator *ExactOp =
3215  dyn_cast<const PossiblyExactOperator>(&I))
3216  exact = ExactOp->isExact();
3217  }
3219  Flags.setExact(exact);
3220  Flags.setNoSignedWrap(nsw);
3221  Flags.setNoUnsignedWrap(nuw);
3222  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3223  Flags);
3224  setValue(&I, Res);
3225 }
3226 
3227 void SelectionDAGBuilder::visitSDiv(const User &I) {
3228  SDValue Op1 = getValue(I.getOperand(0));
3229  SDValue Op2 = getValue(I.getOperand(1));
3230 
3232  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3233  cast<PossiblyExactOperator>(&I)->isExact());
3235  Op2, Flags));
3236 }
3237 
3238 void SelectionDAGBuilder::visitICmp(const User &I) {
3240  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3241  predicate = IC->getPredicate();
3242  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3243  predicate = ICmpInst::Predicate(IC->getPredicate());
3244  SDValue Op1 = getValue(I.getOperand(0));
3245  SDValue Op2 = getValue(I.getOperand(1));
3246  ISD::CondCode Opcode = getICmpCondCode(predicate);
3247 
3248  auto &TLI = DAG.getTargetLoweringInfo();
3249  EVT MemVT =
3250  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3251 
3252  // If a pointer's DAG type is larger than its memory type then the DAG values
3253  // are zero-extended. This breaks signed comparisons so truncate back to the
3254  // underlying type before doing the compare.
3255  if (Op1.getValueType() != MemVT) {
3256  Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3257  Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3258  }
3259 
3261  I.getType());
3262  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3263 }
3264 
3265 void SelectionDAGBuilder::visitFCmp(const User &I) {
3267  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3268  predicate = FC->getPredicate();
3269  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3270  predicate = FCmpInst::Predicate(FC->getPredicate());
3271  SDValue Op1 = getValue(I.getOperand(0));
3272  SDValue Op2 = getValue(I.getOperand(1));
3273 
3274  ISD::CondCode Condition = getFCmpCondCode(predicate);
3275  auto *FPMO = cast<FPMathOperator>(&I);
3276  if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3277  Condition = getFCmpCodeWithoutNaN(Condition);
3278 
3280  Flags.copyFMF(*FPMO);
3281  SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3282 
3284  I.getType());
3285  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3286 }
3287 
3288 // Check if the condition of the select has one use or two users that are both
3289 // selects with the same condition.
3290 static bool hasOnlySelectUsers(const Value *Cond) {
3291  return llvm::all_of(Cond->users(), [](const Value *V) {
3292  return isa<SelectInst>(V);
3293  });
3294 }
3295 
3296 void SelectionDAGBuilder::visitSelect(const User &I) {
3297  SmallVector<EVT, 4> ValueVTs;
3299  ValueVTs);
3300  unsigned NumValues = ValueVTs.size();
3301  if (NumValues == 0) return;
3302 
3303  SmallVector<SDValue, 4> Values(NumValues);
3304  SDValue Cond = getValue(I.getOperand(0));
3305  SDValue LHSVal = getValue(I.getOperand(1));
3306  SDValue RHSVal = getValue(I.getOperand(2));
3307  SmallVector<SDValue, 1> BaseOps(1, Cond);
3308  ISD::NodeType OpCode =
3309  Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3310 
3311  bool IsUnaryAbs = false;
3312  bool Negate = false;
3313 
3315  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3316  Flags.copyFMF(*FPOp);
3317 
3318  // Min/max matching is only viable if all output VTs are the same.
3319  if (all_equal(ValueVTs)) {
3320  EVT VT = ValueVTs[0];
3321  LLVMContext &Ctx = *DAG.getContext();
3322  auto &TLI = DAG.getTargetLoweringInfo();
3323 
3324  // We care about the legality of the operation after it has been type
3325  // legalized.
3326  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3327  VT = TLI.getTypeToTransformTo(Ctx, VT);
3328 
3329  // If the vselect is legal, assume we want to leave this as a vector setcc +
3330  // vselect. Otherwise, if this is going to be scalarized, we want to see if
3331  // min/max is legal on the scalar type.
3332  bool UseScalarMinMax = VT.isVector() &&
3334 
3335  Value *LHS, *RHS;
3336  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3338  switch (SPR.Flavor) {
3339  case SPF_UMAX: Opc = ISD::UMAX; break;
3340  case SPF_UMIN: Opc = ISD::UMIN; break;
3341  case SPF_SMAX: Opc = ISD::SMAX; break;
3342  case SPF_SMIN: Opc = ISD::SMIN; break;
3343  case SPF_FMINNUM:
3344  switch (SPR.NaNBehavior) {
3345  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3346  case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3347  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3348  case SPNB_RETURNS_ANY: {
3350  Opc = ISD::FMINNUM;
3351  else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3352  Opc = ISD::FMINIMUM;
3353  else if (UseScalarMinMax)
3356  break;
3357  }
3358  }
3359  break;
3360  case SPF_FMAXNUM:
3361  switch (SPR.NaNBehavior) {
3362  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3363  case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3364  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3365  case SPNB_RETURNS_ANY:
3366 
3368  Opc = ISD::FMAXNUM;
3369  else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3370  Opc = ISD::FMAXIMUM;
3371  else if (UseScalarMinMax)
3374  break;
3375  }
3376  break;
3377  case SPF_NABS:
3378  Negate = true;
3379  [[fallthrough]];
3380  case SPF_ABS:
3381  IsUnaryAbs = true;
3382  Opc = ISD::ABS;
3383  break;
3384  default: break;
3385  }
3386 
3387  if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3388  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3389  (UseScalarMinMax &&
3390  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3391  // If the underlying comparison instruction is used by any other
3392  // instruction, the consumed instructions won't be destroyed, so it is
3393  // not profitable to convert to a min/max.
3394  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3395  OpCode = Opc;
3396  LHSVal = getValue(LHS);
3397  RHSVal = getValue(RHS);
3398  BaseOps.clear();
3399  }
3400 
3401  if (IsUnaryAbs) {
3402  OpCode = Opc;
3403  LHSVal = getValue(LHS);
3404  BaseOps.clear();
3405  }
3406  }
3407 
3408  if (IsUnaryAbs) {
3409  for (unsigned i = 0; i != NumValues; ++i) {
3410  SDLoc dl = getCurSDLoc();
3411  EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3412  Values[i] =
3413  DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3414  if (Negate)
3415  Values[i] = DAG.getNegative(Values[i], dl, VT);
3416  }
3417  } else {
3418  for (unsigned i = 0; i != NumValues; ++i) {
3419  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3420  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3421  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3422  Values[i] = DAG.getNode(
3423  OpCode, getCurSDLoc(),
3424  LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3425  }
3426  }
3427 
3429  DAG.getVTList(ValueVTs), Values));
3430 }
3431 
3432 void SelectionDAGBuilder::visitTrunc(const User &I) {
3433  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3434  SDValue N = getValue(I.getOperand(0));
3436  I.getType());
3437  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3438 }
3439 
3440 void SelectionDAGBuilder::visitZExt(const User &I) {
3441  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3442  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3443  SDValue N = getValue(I.getOperand(0));
3445  I.getType());
3447 }
3448 
3449 void SelectionDAGBuilder::visitSExt(const User &I) {
3450  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3451  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3452  SDValue N = getValue(I.getOperand(0));
3454  I.getType());
3456 }
3457 
3458 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3459  // FPTrunc is never a no-op cast, no need to check
3460  SDValue N = getValue(I.getOperand(0));
3461  SDLoc dl = getCurSDLoc();
3462  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3463  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3464  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3466  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3467 }
3468 
3469 void SelectionDAGBuilder::visitFPExt(const User &I) {
3470  // FPExt is never a no-op cast, no need to check
3471  SDValue N = getValue(I.getOperand(0));
3473  I.getType());
3474  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3475 }
3476 
3477 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3478  // FPToUI is never a no-op cast, no need to check
3479  SDValue N = getValue(I.getOperand(0));
3481  I.getType());
3482  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3483 }
3484 
3485 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3486  // FPToSI is never a no-op cast, no need to check
3487  SDValue N = getValue(I.getOperand(0));
3489  I.getType());
3490  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3491 }
3492 
3493 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3494  // UIToFP is never a no-op cast, no need to check
3495  SDValue N = getValue(I.getOperand(0));
3497  I.getType());
3498  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3499 }
3500 
3501 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3502  // SIToFP is never a no-op cast, no need to check
3503  SDValue N = getValue(I.getOperand(0));
3505  I.getType());
3506  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3507 }
3508 
3509 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3510  // What to do depends on the size of the integer and the size of the pointer.
3511  // We can either truncate, zero extend, or no-op, accordingly.
3512  SDValue N = getValue(I.getOperand(0));
3513  auto &TLI = DAG.getTargetLoweringInfo();
3515  I.getType());
3516  EVT PtrMemVT =
3517  TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3518  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3519  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3520  setValue(&I, N);
3521 }
3522 
3523 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3524  // What to do depends on the size of the integer and the size of the pointer.
3525  // We can either truncate, zero extend, or no-op, accordingly.
3526  SDValue N = getValue(I.getOperand(0));
3527  auto &TLI = DAG.getTargetLoweringInfo();
3528  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3529  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3530  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3531  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3532  setValue(&I, N);
3533 }
3534 
3535 void SelectionDAGBuilder::visitBitCast(const User &I) {
3536  SDValue N = getValue(I.getOperand(0));
3537  SDLoc dl = getCurSDLoc();
3539  I.getType());
3540 
3541  // BitCast assures us that source and destination are the same size so this is
3542  // either a BITCAST or a no-op.
3543  if (DestVT != N.getValueType())
3545  DestVT, N)); // convert types.
3546  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3547  // might fold any kind of constant expression to an integer constant and that
3548  // is not what we are looking for. Only recognize a bitcast of a genuine
3549  // constant integer as an opaque constant.
3550  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3551  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3552  /*isOpaque*/true));
3553  else
3554  setValue(&I, N); // noop cast.
3555 }
3556 
3557 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3558  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3559  const Value *SV = I.getOperand(0);
3560  SDValue N = getValue(SV);
3561  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3562 
3563  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3564  unsigned DestAS = I.getType()->getPointerAddressSpace();
3565 
3566  if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3567  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3568 
3569  setValue(&I, N);
3570 }
3571 
3572 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3573  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3574  SDValue InVec = getValue(I.getOperand(0));
3575  SDValue InVal = getValue(I.getOperand(1));
3576  SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3579  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3580  InVec, InVal, InIdx));
3581 }
3582 
3583 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3584  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3585  SDValue InVec = getValue(I.getOperand(0));
3586  SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3589  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3590  InVec, InIdx));
3591 }
3592 
3593 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3594  SDValue Src1 = getValue(I.getOperand(0));
3595  SDValue Src2 = getValue(I.getOperand(1));
3597  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3598  Mask = SVI->getShuffleMask();
3599  else
3600  Mask = cast<ConstantExpr>(I).getShuffleMask();
3601  SDLoc DL = getCurSDLoc();
3602  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3603  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3604  EVT SrcVT = Src1.getValueType();
3605 
3606  if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3607  VT.isScalableVector()) {
3608  // Canonical splat form of first element of first input vector.
3609  SDValue FirstElt =
3612  setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3613  return;
3614  }
3615 
3616  // For now, we only handle splats for scalable vectors.
3617  // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3618  // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3619  assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3620 
3621  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3622  unsigned MaskNumElts = Mask.size();
3623 
3624  if (SrcNumElts == MaskNumElts) {
3625  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3626  return;
3627  }
3628 
3629  // Normalize the shuffle vector since mask and vector length don't match.
3630  if (SrcNumElts < MaskNumElts) {
3631  // Mask is longer than the source vectors. We can use concatenate vector to
3632  // make the mask and vectors lengths match.
3633 
3634  if (MaskNumElts % SrcNumElts == 0) {
3635  // Mask length is a multiple of the source vector length.
3636  // Check if the shuffle is some kind of concatenation of the input
3637  // vectors.
3638  unsigned NumConcat = MaskNumElts / SrcNumElts;
3639  bool IsConcat = true;
3640  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3641  for (unsigned i = 0; i != MaskNumElts; ++i) {
3642  int Idx = Mask[i];
3643  if (Idx < 0)
3644  continue;
3645  // Ensure the indices in each SrcVT sized piece are sequential and that
3646  // the same source is used for the whole piece.
3647  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3648  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3649  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3650  IsConcat = false;
3651  break;
3652  }
3653  // Remember which source this index came from.
3654  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3655  }
3656 
3657  // The shuffle is concatenating multiple vectors together. Just emit
3658  // a CONCAT_VECTORS operation.
3659  if (IsConcat) {
3660  SmallVector<SDValue, 8> ConcatOps;
3661  for (auto Src : ConcatSrcs) {
3662  if (Src < 0)
3663  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3664  else if (Src == 0)
3665  ConcatOps.push_back(Src1);
3666  else
3667  ConcatOps.push_back(Src2);
3668  }
3669  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3670  return;
3671  }
3672  }
3673 
3674  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3675  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3676  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3677  PaddedMaskNumElts);
3678 
3679  // Pad both vectors with undefs to make them the same length as the mask.
3680  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3681 
3682  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3683  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3684  MOps1[0] = Src1;
3685  MOps2[0] = Src2;
3686 
3687  Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3688  Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3689 
3690  // Readjust mask for new input vector length.
3691  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3692  for (unsigned i = 0; i != MaskNumElts; ++i) {
3693  int Idx = Mask[i];
3694  if (Idx >= (int)SrcNumElts)
3695  Idx -= SrcNumElts - PaddedMaskNumElts;
3696  MappedOps[i] = Idx;
3697  }
3698 
3699  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3700 
3701  // If the concatenated vector was padded, extract a subvector with the
3702  // correct number of elements.
3703  if (MaskNumElts != PaddedMaskNumElts)
3704  Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3706 
3707  setValue(&I, Result);
3708  return;
3709  }
3710 
3711  if (SrcNumElts > MaskNumElts) {
3712  // Analyze the access pattern of the vector to see if we can extract
3713  // two subvectors and do the shuffle.
3714  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3715  bool CanExtract = true;
3716  for (int Idx : Mask) {
3717  unsigned Input = 0;
3718  if (Idx < 0)
3719  continue;
3720 
3721  if (Idx >= (int)SrcNumElts) {
3722  Input = 1;
3723  Idx -= SrcNumElts;
3724  }
3725 
3726  // If all the indices come from the same MaskNumElts sized portion of
3727  // the sources we can use extract. Also make sure the extract wouldn't
3728  // extract past the end of the source.
3729  int NewStartIdx = alignDown(Idx, MaskNumElts);
3730  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3731  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3732  CanExtract = false;
3733  // Make sure we always update StartIdx as we use it to track if all
3734  // elements are undef.
3735  StartIdx[Input] = NewStartIdx;
3736  }
3737 
3738  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3739  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3740  return;
3741  }
3742  if (CanExtract) {
3743  // Extract appropriate subvector and generate a vector shuffle
3744  for (unsigned Input = 0; Input < 2; ++Input) {
3745  SDValue &Src = Input == 0 ? Src1 : Src2;
3746  if (StartIdx[Input] < 0)
3747  Src = DAG.getUNDEF(VT);
3748  else {
3749  Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3750  DAG.getVectorIdxConstant(StartIdx[Input], DL));
3751  }
3752  }
3753 
3754  // Calculate new mask.
3755  SmallVector<int, 8> MappedOps(Mask);
3756  for (int &Idx : MappedOps) {
3757  if (Idx >= (int)SrcNumElts)
3758  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3759  else if (Idx >= 0)
3760  Idx -= StartIdx[0];
3761  }
3762 
3763  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3764  return;
3765  }
3766  }
3767 
3768  // We can't use either concat vectors or extract subvectors so fall back to
3769  // replacing the shuffle with extract and build vector.
3770  // to insert and build vector.
3771  EVT EltVT = VT.getVectorElementType();
3773  for (int Idx : Mask) {
3774  SDValue Res;
3775 
3776  if (Idx < 0) {
3777  Res = DAG.getUNDEF(EltVT);
3778  } else {
3779  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3780  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3781 
3782  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3783  DAG.getVectorIdxConstant(Idx, DL));
3784  }
3785 
3786  Ops.push_back(Res);
3787  }
3788 
3789  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3790 }
3791 
3792 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3793  ArrayRef<unsigned> Indices = I.getIndices();
3794  const Value *Op0 = I.getOperand(0);
3795  const Value *Op1 = I.getOperand(1);
3796  Type *AggTy = I.getType();
3797  Type *ValTy = Op1->getType();
3798  bool IntoUndef = isa<UndefValue>(Op0);
3799  bool FromUndef = isa<UndefValue>(Op1);
3800 
3801  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3802 
3803  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3804  SmallVector<EVT, 4> AggValueVTs;
3805  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3806  SmallVector<EVT, 4> ValValueVTs;
3807  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3808 
3809  unsigned NumAggValues = AggValueVTs.size();
3810  unsigned NumValValues = ValValueVTs.size();
3811  SmallVector<SDValue, 4> Values(NumAggValues);
3812 
3813  // Ignore an insertvalue that produces an empty object
3814  if (!NumAggValues) {
3816  return;
3817  }
3818 
3819  SDValue Agg = getValue(Op0);
3820  unsigned i = 0;
3821  // Copy the beginning value(s) from the original aggregate.
3822  for (; i != LinearIndex; ++i)
3823  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3824  SDValue(Agg.getNode(), Agg.getResNo() + i);
3825  // Copy values from the inserted value(s).
3826  if (NumValValues) {
3827  SDValue Val = getValue(Op1);
3828  for (; i != LinearIndex + NumValValues; ++i)
3829  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3830  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3831  }
3832  // Copy remaining value(s) from the original aggregate.
3833  for (; i != NumAggValues; ++i)
3834  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3835  SDValue(Agg.getNode(), Agg.getResNo() + i);
3836 
3838  DAG.getVTList(AggValueVTs), Values));
3839 }
3840 
3841 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3842  ArrayRef<unsigned> Indices = I.getIndices();
3843  const Value *Op0 = I.getOperand(0);
3844  Type *AggTy = Op0->getType();
3845  Type *ValTy = I.getType();
3846  bool OutOfUndef = isa<UndefValue>(Op0);
3847 
3848  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3849 
3850  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3851  SmallVector<EVT, 4> ValValueVTs;
3852  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3853 
3854  unsigned NumValValues = ValValueVTs.size();
3855 
3856  // Ignore a extractvalue that produces an empty object
3857  if (!NumValValues) {
3859  return;
3860  }
3861 
3862  SmallVector<SDValue, 4> Values(NumValValues);
3863 
3864  SDValue Agg = getValue(Op0);
3865  // Copy out the selected value(s).
3866  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3867  Values[i - LinearIndex] =
3868  OutOfUndef ?
3869  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3870  SDValue(Agg.getNode(), Agg.getResNo() + i);
3871 
3873  DAG.getVTList(ValValueVTs), Values));
3874 }
3875 
3876 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3877  Value *Op0 = I.getOperand(0);
3878  // Note that the pointer operand may be a vector of pointers. Take the scalar
3879  // element which holds a pointer.
3880  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3881  SDValue N = getValue(Op0);
3882  SDLoc dl = getCurSDLoc();
3883  auto &TLI = DAG.getTargetLoweringInfo();
3884 
3885  // Normalize Vector GEP - all scalar operands should be converted to the
3886  // splat vector.
3887  bool IsVectorGEP = I.getType()->isVectorTy();
3888  ElementCount VectorElementCount =
3889  IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3891 
3892  if (IsVectorGEP && !N.getValueType().isVector()) {
3894  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3895  N = DAG.getSplat(VT, dl, N);
3896  }
3897 
3898  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3899  GTI != E; ++GTI) {
3900  const Value *Idx = GTI.getOperand();
3901  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3902  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3903  if (Field) {
3904  // N = N + Offset
3905  uint64_t Offset =
3907 
3908  // In an inbounds GEP with an offset that is nonnegative even when
3909  // interpreted as signed, assume there is no unsigned overflow.
3911  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3912  Flags.setNoUnsignedWrap(true);
3913 
3914  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3915  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3916  }
3917  } else {
3918  // IdxSize is the width of the arithmetic according to IR semantics.
3919  // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3920  // (and fix up the result later).
3921  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3922  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3923  TypeSize ElementSize =
3924  DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3925  // We intentionally mask away the high bits here; ElementSize may not
3926  // fit in IdxTy.
3927  APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3928  bool ElementScalable = ElementSize.isScalable();
3929 
3930  // If this is a scalar constant or a splat vector of constants,
3931  // handle it quickly.
3932  const auto *C = dyn_cast<Constant>(Idx);
3933  if (C && isa<VectorType>(C->getType()))
3934  C = C->getSplatValue();
3935 
3936  const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3937  if (CI && CI->isZero())
3938  continue;
3939  if (CI && !ElementScalable) {
3940  APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3942  SDValue OffsVal;
3943  if (IsVectorGEP)
3944  OffsVal = DAG.getConstant(
3945  Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3946  else
3947  OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3948 
3949  // In an inbounds GEP with an offset that is nonnegative even when
3950  // interpreted as signed, assume there is no unsigned overflow.
3952  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3953  Flags.setNoUnsignedWrap(true);
3954 
3955  OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3956 
3957  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3958  continue;
3959  }
3960 
3961  // N = N + Idx * ElementMul;
3962  SDValue IdxN = getValue(Idx);
3963 
3964  if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3965  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3966  VectorElementCount);
3967  IdxN = DAG.getSplat(VT, dl, IdxN);
3968  }
3969 
3970  // If the index is smaller or larger than intptr_t, truncate or extend
3971  // it.
3972  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3973 
3974  if (ElementScalable) {
3975  EVT VScaleTy = N.getValueType().getScalarType();
3976  SDValue VScale = DAG.getNode(
3977  ISD::VSCALE, dl, VScaleTy,
3978  DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3979  if (IsVectorGEP)
3980  VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3981  IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3982  } else {
3983  // If this is a multiply by a power of two, turn it into a shl
3984  // immediately. This is a very common case.
3985  if (ElementMul != 1) {
3986  if (ElementMul.isPowerOf2()) {
3987  unsigned Amt = ElementMul.logBase2();
3988  IdxN = DAG.getNode(ISD::SHL, dl,
3989  N.getValueType(), IdxN,
3990  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3991  } else {
3992  SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3993  IdxN.getValueType());
3994  IdxN = DAG.getNode(ISD::MUL, dl,
3995  N.getValueType(), IdxN, Scale);
3996  }
3997  }
3998  }
3999 
4000  N = DAG.getNode(ISD::ADD, dl,
4001  N.getValueType(), N, IdxN);
4002  }
4003  }
4004 
4005  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4006  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4007  if (IsVectorGEP) {
4008  PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4009  PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4010  }
4011 
4012  if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4013  N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4014 
4015  setValue(&I, N);
4016 }
4017 
4018 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4019  // If this is a fixed sized alloca in the entry block of the function,
4020  // allocate it statically on the stack.
4021  if (FuncInfo.StaticAllocaMap.count(&I))
4022  return; // getValue will auto-populate this.
4023 
4024  SDLoc dl = getCurSDLoc();
4025  Type *Ty = I.getAllocatedType();
4026  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4027  auto &DL = DAG.getDataLayout();
4028  TypeSize TySize = DL.getTypeAllocSize(Ty);
4029  MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4030 
4031  SDValue AllocSize = getValue(I.getArraySize());
4032 
4033  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());
4034  if (AllocSize.getValueType() != IntPtr)
4035  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4036 
4037  if (TySize.isScalable())
4038  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4039  DAG.getVScale(dl, IntPtr,
4040  APInt(IntPtr.getScalarSizeInBits(),
4041  TySize.getKnownMinValue())));
4042  else
4043  AllocSize =
4044  DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4045  DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4046 
4047  // Handle alignment. If the requested alignment is less than or equal to
4048  // the stack alignment, ignore it. If the size is greater than or equal to
4049  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4051  if (*Alignment <= StackAlign)
4052  Alignment = std::nullopt;
4053 
4054  const uint64_t StackAlignMask = StackAlign.value() - 1U;
4055  // Round the size of the allocation up to the stack alignment size
4056  // by add SA-1 to the size. This doesn't overflow because we're computing
4057  // an address inside an alloca.
4059  Flags.setNoUnsignedWrap(true);
4060  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4061  DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4062 
4063  // Mask out the low bits for alignment purposes.
4064  AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4065  DAG.getConstant(~StackAlignMask, dl, IntPtr));
4066 
4067  SDValue Ops[] = {
4068  getRoot(), AllocSize,
4069  DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4070  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4071  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4072  setValue(&I, DSA);
4073  DAG.setRoot(DSA.getValue(1));
4074 
4076 }
4077 
4078 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4079  if (I.isAtomic())
4080  return visitAtomicLoad(I);
4081 
4082  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4083  const Value *SV = I.getOperand(0);
4084  if (TLI.supportSwiftError()) {
4085  // Swifterror values can come from either a function parameter with
4086  // swifterror attribute or an alloca with swifterror attribute.
4087  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4088  if (Arg->hasSwiftErrorAttr())
4089  return visitLoadFromSwiftError(I);
4090  }
4091 
4092  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4093  if (Alloca->isSwiftError())
4094  return visitLoadFromSwiftError(I);
4095  }
4096  }
4097 
4098  SDValue Ptr = getValue(SV);
4099 
4100  Type *Ty = I.getType();
4101  SmallVector<EVT, 4> ValueVTs, MemVTs;
4103  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4104  unsigned NumValues = ValueVTs.size();
4105  if (NumValues == 0)
4106  return;
4107 
4108  Align Alignment = I.getAlign();
4109  AAMDNodes AAInfo = I.getAAMetadata();
4110  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4111  bool isVolatile = I.isVolatile();
4112  MachineMemOperand::Flags MMOFlags =
4114 
4115  SDValue Root;
4116  bool ConstantMemory = false;
4117  if (isVolatile)
4118  // Serialize volatile loads with other side effects.
4119  Root = getRoot();
4120  else if (NumValues > MaxParallelChains)
4121  Root = getMemoryRoot();
4122  else if (AA &&
4124  SV,
4126  AAInfo))) {
4127  // Do not serialize (non-volatile) loads of constant memory with anything.
4128  Root = DAG.getEntryNode();
4129  ConstantMemory = true;
4130  MMOFlags |= MachineMemOperand::MOInvariant;
4131  } else {
4132  // Do not serialize non-volatile loads against each other.
4133  Root = DAG.getRoot();
4134  }
4135 
4136  if (isDereferenceableAndAlignedPointer(SV, Ty, Alignment, DAG.getDataLayout(),
4137  &I, AC, nullptr, LibInfo))
4139 
4140  SDLoc dl = getCurSDLoc();
4141 
4142  if (isVolatile)
4143  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4144 
4145  // An aggregate load cannot wrap around the address space, so offsets to its
4146  // parts don't wrap either.
4148  Flags.setNoUnsignedWrap(true);
4149 
4150  SmallVector<SDValue, 4> Values(NumValues);
4152  EVT PtrVT = Ptr.getValueType();
4153 
4154  unsigned ChainI = 0;
4155  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4156  // Serializing loads here may result in excessive register pressure, and
4157  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4158  // could recover a bit by hoisting nodes upward in the chain by recognizing
4159  // they are side-effect free or do not alias. The optimizer should really
4160  // avoid this case by converting large object/array copies to llvm.memcpy
4161  // (MaxParallelChains should always remain as failsafe).
4162  if (ChainI == MaxParallelChains) {
4163  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4165  makeArrayRef(Chains.data(), ChainI));
4166  Root = Chain;
4167  ChainI = 0;
4168  }
4169  SDValue A = DAG.getNode(ISD::ADD, dl,
4170  PtrVT, Ptr,
4171  DAG.getConstant(Offsets[i], dl, PtrVT),
4172  Flags);
4173 
4174  SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4175  MachinePointerInfo(SV, Offsets[i]), Alignment,
4176  MMOFlags, AAInfo, Ranges);
4177  Chains[ChainI] = L.getValue(1);
4178 
4179  if (MemVTs[i] != ValueVTs[i])
4180  L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4181 
4182  Values[i] = L;
4183  }
4184 
4185  if (!ConstantMemory) {
4187  makeArrayRef(Chains.data(), ChainI));
4188  if (isVolatile)
4189  DAG.setRoot(Chain);
4190  else
4191  PendingLoads.push_back(Chain);
4192  }
4193 
4195  DAG.getVTList(ValueVTs), Values));
4196 }
4197 
4198 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4200  "call visitStoreToSwiftError when backend supports swifterror");
4201 
4202  SmallVector<EVT, 4> ValueVTs;
4204  const Value *SrcV = I.getOperand(0);
4206  SrcV->getType(), ValueVTs, &Offsets);
4207  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4208  "expect a single EVT for swifterror");
4209 
4210  SDValue Src = getValue(SrcV);
4211  // Create a virtual register, then update the virtual register.
4212  Register VReg =
4213  SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4214  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4215  // Chain can be getRoot or getControlRoot.
4216  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4217  SDValue(Src.getNode(), Src.getResNo()));
4218  DAG.setRoot(CopyNode);
4219 }
4220 
4221 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4223  "call visitLoadFromSwiftError when backend supports swifterror");
4224 
4225  assert(!I.isVolatile() &&
4226  !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4227  !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4228  "Support volatile, non temporal, invariant for load_from_swift_error");
4229 
4230  const Value *SV = I.getOperand(0);
4231  Type *Ty = I.getType();
4232  assert(
4233  (!AA ||
4236  I.getAAMetadata()))) &&
4237  "load_from_swift_error should not be constant memory");
4238 
4239  SmallVector<EVT, 4> ValueVTs;
4242  ValueVTs, &Offsets);
4243  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4244  "expect a single EVT for swifterror");
4245 
4246  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4248  getRoot(), getCurSDLoc(),
4249  SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4250 
4251  setValue(&I, L);
4252 }
4253 
4254 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4255  if (I.isAtomic())
4256  return visitAtomicStore(I);
4257 
4258  const Value *SrcV = I.getOperand(0);
4259  const Value *PtrV = I.getOperand(1);
4260 
4261  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4262  if (TLI.supportSwiftError()) {
4263  // Swifterror values can come from either a function parameter with
4264  // swifterror attribute or an alloca with swifterror attribute.
4265  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4266  if (Arg->hasSwiftErrorAttr())
4267  return visitStoreToSwiftError(I);
4268  }
4269 
4270  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4271  if (Alloca->isSwiftError())
4272  return visitStoreToSwiftError(I);
4273  }
4274  }
4275 
4276  SmallVector<EVT, 4> ValueVTs, MemVTs;
4279  SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4280  unsigned NumValues = ValueVTs.size();
4281  if (NumValues == 0)
4282  return;
4283 
4284  // Get the lowered operands. Note that we do this after
4285  // checking if NumResults is zero, because with zero results
4286  // the operands won't have values in the map.
4287  SDValue Src = getValue(SrcV);
4288  SDValue Ptr = getValue(PtrV);
4289 
4290  SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4292  SDLoc dl = getCurSDLoc();
4293  Align Alignment = I.getAlign();
4294  AAMDNodes AAInfo = I.getAAMetadata();
4295 
4296  auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4297 
4298  // An aggregate load cannot wrap around the address space, so offsets to its
4299  // parts don't wrap either.
4301  Flags.setNoUnsignedWrap(true);
4302 
4303  unsigned ChainI = 0;
4304  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4305  // See visitLoad comments.
4306  if (ChainI == MaxParallelChains) {
4308  makeArrayRef(Chains.data(), ChainI));
4309  Root = Chain;
4310  ChainI = 0;
4311  }
4312  SDValue Add =
4314  SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4315  if (MemVTs[i] != ValueVTs[i])
4316  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4317  SDValue St =
4318  DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4319  Alignment, MMOFlags, AAInfo);
4320  Chains[ChainI] = St;
4321  }
4322 
4323  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4324  makeArrayRef(Chains.data(), ChainI));
4325  setValue(&I, StoreNode);
4326  DAG.setRoot(StoreNode);
4327 }
4328 
4329 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4330  bool IsCompressing) {
4331  SDLoc sdl = getCurSDLoc();
4332 
4333  auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4334  MaybeAlign &Alignment) {
4335  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4336  Src0 = I.getArgOperand(0);
4337  Ptr = I.getArgOperand(1);
4338  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4339  Mask = I.getArgOperand(3);
4340  };
4341  auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4342  MaybeAlign &Alignment) {
4343  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4344  Src0 = I.getArgOperand(0);
4345  Ptr = I.getArgOperand(1);
4346  Mask = I.getArgOperand(2);
4347  Alignment = std::nullopt;
4348  };
4349 
4350  Value *PtrOperand, *MaskOperand, *Src0Operand;
4351  MaybeAlign Alignment;
4352  if (IsCompressing)
4353  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4354  else
4355  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4356 
4357  SDValue Ptr = getValue(PtrOperand);
4358  SDValue Src0 = getValue(Src0Operand);
4359  SDValue Mask = getValue(MaskOperand);
4360  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4361 
4362  EVT VT = Src0.getValueType();
4363  if (!Alignment)
4364  Alignment = DAG.getEVTAlign(VT);
4365 
4368  MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
4369  SDValue StoreNode =
4370  DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4371  ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4372  DAG.setRoot(StoreNode);
4373  setValue(&I, StoreNode);
4374 }
4375 
4376 // Get a uniform base for the Gather/Scatter intrinsic.
4377 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4378 // We try to represent it as a base pointer + vector of indices.
4379 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4380 // The first operand of the GEP may be a single pointer or a vector of pointers
4381 // Example:
4382 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4383 // or
4384 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4385 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4386 //
4387 // When the first GEP operand is a single pointer - it is the uniform base we
4388 // are looking for. If first operand of the GEP is a splat vector - we
4389 // extract the splat value and use it as a uniform base.
4390 // In all other cases the function returns 'false'.
4391 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4392  ISD::MemIndexType &IndexType, SDValue &Scale,
4393  SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4394