LLVM  6.0.0svn
VectorUtils.cpp
Go to the documentation of this file.
1 //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines vectorizer utilities.
11 //
12 //===----------------------------------------------------------------------===//
13 
17 #include "llvm/Analysis/LoopInfo.h"
22 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/IRBuilder.h"
25 #include "llvm/IR/PatternMatch.h"
26 #include "llvm/IR/Value.h"
27 
28 using namespace llvm;
29 using namespace llvm::PatternMatch;
30 
31 /// \brief Identify if the intrinsic is trivially vectorizable.
32 /// This method returns true if the intrinsic's argument types are all
33 /// scalars for the scalar form of the intrinsic and all vectors for
34 /// the vector form of the intrinsic.
36  switch (ID) {
37  case Intrinsic::sqrt:
38  case Intrinsic::sin:
39  case Intrinsic::cos:
40  case Intrinsic::exp:
41  case Intrinsic::exp2:
42  case Intrinsic::log:
43  case Intrinsic::log10:
44  case Intrinsic::log2:
45  case Intrinsic::fabs:
46  case Intrinsic::minnum:
47  case Intrinsic::maxnum:
48  case Intrinsic::copysign:
49  case Intrinsic::floor:
50  case Intrinsic::ceil:
51  case Intrinsic::trunc:
52  case Intrinsic::rint:
53  case Intrinsic::nearbyint:
54  case Intrinsic::round:
55  case Intrinsic::bswap:
56  case Intrinsic::bitreverse:
57  case Intrinsic::ctpop:
58  case Intrinsic::pow:
59  case Intrinsic::fma:
60  case Intrinsic::fmuladd:
61  case Intrinsic::ctlz:
62  case Intrinsic::cttz:
63  case Intrinsic::powi:
64  return true;
65  default:
66  return false;
67  }
68 }
69 
70 /// \brief Identifies if the intrinsic has a scalar operand. It check for
71 /// ctlz,cttz and powi special intrinsics whose argument is scalar.
73  unsigned ScalarOpdIdx) {
74  switch (ID) {
75  case Intrinsic::ctlz:
76  case Intrinsic::cttz:
77  case Intrinsic::powi:
78  return (ScalarOpdIdx == 1);
79  default:
80  return false;
81  }
82 }
83 
84 /// \brief Returns intrinsic ID for call.
85 /// For the input call instruction it finds mapping intrinsic and returns
86 /// its ID, in case it does not found it return not_intrinsic.
88  const TargetLibraryInfo *TLI) {
90  if (ID == Intrinsic::not_intrinsic)
92 
93  if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
94  ID == Intrinsic::lifetime_end || ID == Intrinsic::assume ||
95  ID == Intrinsic::sideeffect)
96  return ID;
98 }
99 
100 /// \brief Find the operand of the GEP that should be checked for consecutive
101 /// stores. This ignores trailing indices that have no effect on the final
102 /// pointer.
104  const DataLayout &DL = Gep->getModule()->getDataLayout();
105  unsigned LastOperand = Gep->getNumOperands() - 1;
106  unsigned GEPAllocSize = DL.getTypeAllocSize(Gep->getResultElementType());
107 
108  // Walk backwards and try to peel off zeros.
109  while (LastOperand > 1 && match(Gep->getOperand(LastOperand), m_Zero())) {
110  // Find the type we're currently indexing into.
111  gep_type_iterator GEPTI = gep_type_begin(Gep);
112  std::advance(GEPTI, LastOperand - 2);
113 
114  // If it's a type with the same allocation size as the result of the GEP we
115  // can peel off the zero index.
116  if (DL.getTypeAllocSize(GEPTI.getIndexedType()) != GEPAllocSize)
117  break;
118  --LastOperand;
119  }
120 
121  return LastOperand;
122 }
123 
124 /// \brief If the argument is a GEP, then returns the operand identified by
125 /// getGEPInductionOperand. However, if there is some other non-loop-invariant
126 /// operand, it returns that instead.
129  if (!GEP)
130  return Ptr;
131 
132  unsigned InductionOperand = getGEPInductionOperand(GEP);
133 
134  // Check that all of the gep indices are uniform except for our induction
135  // operand.
136  for (unsigned i = 0, e = GEP->getNumOperands(); i != e; ++i)
137  if (i != InductionOperand &&
138  !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(i)), Lp))
139  return Ptr;
140  return GEP->getOperand(InductionOperand);
141 }
142 
143 /// \brief If a value has only one user that is a CastInst, return it.
145  Value *UniqueCast = nullptr;
146  for (User *U : Ptr->users()) {
147  CastInst *CI = dyn_cast<CastInst>(U);
148  if (CI && CI->getType() == Ty) {
149  if (!UniqueCast)
150  UniqueCast = CI;
151  else
152  return nullptr;
153  }
154  }
155  return UniqueCast;
156 }
157 
158 /// \brief Get the stride of a pointer access in a loop. Looks for symbolic
159 /// strides "a[i*stride]". Returns the symbolic stride, or null otherwise.
161  auto *PtrTy = dyn_cast<PointerType>(Ptr->getType());
162  if (!PtrTy || PtrTy->isAggregateType())
163  return nullptr;
164 
165  // Try to remove a gep instruction to make the pointer (actually index at this
166  // point) easier analyzable. If OrigPtr is equal to Ptr we are analzying the
167  // pointer, otherwise, we are analyzing the index.
168  Value *OrigPtr = Ptr;
169 
170  // The size of the pointer access.
171  int64_t PtrAccessSize = 1;
172 
173  Ptr = stripGetElementPtr(Ptr, SE, Lp);
174  const SCEV *V = SE->getSCEV(Ptr);
175 
176  if (Ptr != OrigPtr)
177  // Strip off casts.
178  while (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V))
179  V = C->getOperand();
180 
181  const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V);
182  if (!S)
183  return nullptr;
184 
185  V = S->getStepRecurrence(*SE);
186  if (!V)
187  return nullptr;
188 
189  // Strip off the size of access multiplication if we are still analyzing the
190  // pointer.
191  if (OrigPtr == Ptr) {
192  if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
193  if (M->getOperand(0)->getSCEVType() != scConstant)
194  return nullptr;
195 
196  const APInt &APStepVal = cast<SCEVConstant>(M->getOperand(0))->getAPInt();
197 
198  // Huge step value - give up.
199  if (APStepVal.getBitWidth() > 64)
200  return nullptr;
201 
202  int64_t StepVal = APStepVal.getSExtValue();
203  if (PtrAccessSize != StepVal)
204  return nullptr;
205  V = M->getOperand(1);
206  }
207  }
208 
209  // Strip off casts.
210  Type *StripedOffRecurrenceCast = nullptr;
211  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V)) {
212  StripedOffRecurrenceCast = C->getType();
213  V = C->getOperand();
214  }
215 
216  // Look for the loop invariant symbolic value.
217  const SCEVUnknown *U = dyn_cast<SCEVUnknown>(V);
218  if (!U)
219  return nullptr;
220 
221  Value *Stride = U->getValue();
222  if (!Lp->isLoopInvariant(Stride))
223  return nullptr;
224 
225  // If we have stripped off the recurrence cast we have to make sure that we
226  // return the value that is used in this loop so that we can replace it later.
227  if (StripedOffRecurrenceCast)
228  Stride = getUniqueCastUse(Stride, Lp, StripedOffRecurrenceCast);
229 
230  return Stride;
231 }
232 
233 /// \brief Given a vector and an element number, see if the scalar value is
234 /// already around as a register, for example if it were inserted then extracted
235 /// from the vector.
236 Value *llvm::findScalarElement(Value *V, unsigned EltNo) {
237  assert(V->getType()->isVectorTy() && "Not looking at a vector?");
238  VectorType *VTy = cast<VectorType>(V->getType());
239  unsigned Width = VTy->getNumElements();
240  if (EltNo >= Width) // Out of range access.
241  return UndefValue::get(VTy->getElementType());
242 
243  if (Constant *C = dyn_cast<Constant>(V))
244  return C->getAggregateElement(EltNo);
245 
246  if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) {
247  // If this is an insert to a variable element, we don't know what it is.
248  if (!isa<ConstantInt>(III->getOperand(2)))
249  return nullptr;
250  unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
251 
252  // If this is an insert to the element we are looking for, return the
253  // inserted value.
254  if (EltNo == IIElt)
255  return III->getOperand(1);
256 
257  // Otherwise, the insertelement doesn't modify the value, recurse on its
258  // vector input.
259  return findScalarElement(III->getOperand(0), EltNo);
260  }
261 
262  if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) {
263  unsigned LHSWidth = SVI->getOperand(0)->getType()->getVectorNumElements();
264  int InEl = SVI->getMaskValue(EltNo);
265  if (InEl < 0)
266  return UndefValue::get(VTy->getElementType());
267  if (InEl < (int)LHSWidth)
268  return findScalarElement(SVI->getOperand(0), InEl);
269  return findScalarElement(SVI->getOperand(1), InEl - LHSWidth);
270  }
271 
272  // Extract a value from a vector add operation with a constant zero.
273  Value *Val = nullptr; Constant *Con = nullptr;
274  if (match(V, m_Add(m_Value(Val), m_Constant(Con))))
275  if (Constant *Elt = Con->getAggregateElement(EltNo))
276  if (Elt->isNullValue())
277  return findScalarElement(Val, EltNo);
278 
279  // Otherwise, we don't know.
280  return nullptr;
281 }
282 
283 /// \brief Get splat value if the input is a splat vector or return nullptr.
284 /// This function is not fully general. It checks only 2 cases:
285 /// the input value is (1) a splat constants vector or (2) a sequence
286 /// of instructions that broadcast a single value into a vector.
287 ///
289 
290  if (auto *C = dyn_cast<Constant>(V))
291  if (isa<VectorType>(V->getType()))
292  return C->getSplatValue();
293 
294  auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V);
295  if (!ShuffleInst)
296  return nullptr;
297  // All-zero (or undef) shuffle mask elements.
298  for (int MaskElt : ShuffleInst->getShuffleMask())
299  if (MaskElt != 0 && MaskElt != -1)
300  return nullptr;
301  // The first shuffle source is 'insertelement' with index 0.
302  auto *InsertEltInst =
303  dyn_cast<InsertElementInst>(ShuffleInst->getOperand(0));
304  if (!InsertEltInst || !isa<ConstantInt>(InsertEltInst->getOperand(2)) ||
305  !cast<ConstantInt>(InsertEltInst->getOperand(2))->isZero())
306  return nullptr;
307 
308  return InsertEltInst->getOperand(1);
309 }
310 
313  const TargetTransformInfo *TTI) {
314 
315  // DemandedBits will give us every value's live-out bits. But we want
316  // to ensure no extra casts would need to be inserted, so every DAG
317  // of connected values must have the same minimum bitwidth.
319  SmallVector<Value *, 16> Worklist;
321  SmallPtrSet<Value *, 16> Visited;
323  SmallPtrSet<Instruction *, 4> InstructionSet;
325 
326  // Determine the roots. We work bottom-up, from truncs or icmps.
327  bool SeenExtFromIllegalType = false;
328  for (auto *BB : Blocks)
329  for (auto &I : *BB) {
330  InstructionSet.insert(&I);
331 
332  if (TTI && (isa<ZExtInst>(&I) || isa<SExtInst>(&I)) &&
333  !TTI->isTypeLegal(I.getOperand(0)->getType()))
334  SeenExtFromIllegalType = true;
335 
336  // Only deal with non-vector integers up to 64-bits wide.
337  if ((isa<TruncInst>(&I) || isa<ICmpInst>(&I)) &&
338  !I.getType()->isVectorTy() &&
339  I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) {
340  // Don't make work for ourselves. If we know the loaded type is legal,
341  // don't add it to the worklist.
342  if (TTI && isa<TruncInst>(&I) && TTI->isTypeLegal(I.getType()))
343  continue;
344 
345  Worklist.push_back(&I);
346  Roots.insert(&I);
347  }
348  }
349  // Early exit.
350  if (Worklist.empty() || (TTI && !SeenExtFromIllegalType))
351  return MinBWs;
352 
353  // Now proceed breadth-first, unioning values together.
354  while (!Worklist.empty()) {
355  Value *Val = Worklist.pop_back_val();
356  Value *Leader = ECs.getOrInsertLeaderValue(Val);
357 
358  if (Visited.count(Val))
359  continue;
360  Visited.insert(Val);
361 
362  // Non-instructions terminate a chain successfully.
363  if (!isa<Instruction>(Val))
364  continue;
365  Instruction *I = cast<Instruction>(Val);
366 
367  // If we encounter a type that is larger than 64 bits, we can't represent
368  // it so bail out.
369  if (DB.getDemandedBits(I).getBitWidth() > 64)
371 
372  uint64_t V = DB.getDemandedBits(I).getZExtValue();
373  DBits[Leader] |= V;
374  DBits[I] = V;
375 
376  // Casts, loads and instructions outside of our range terminate a chain
377  // successfully.
378  if (isa<SExtInst>(I) || isa<ZExtInst>(I) || isa<LoadInst>(I) ||
379  !InstructionSet.count(I))
380  continue;
381 
382  // Unsafe casts terminate a chain unsuccessfully. We can't do anything
383  // useful with bitcasts, ptrtoints or inttoptrs and it'd be unsafe to
384  // transform anything that relies on them.
385  if (isa<BitCastInst>(I) || isa<PtrToIntInst>(I) || isa<IntToPtrInst>(I) ||
386  !I->getType()->isIntegerTy()) {
387  DBits[Leader] |= ~0ULL;
388  continue;
389  }
390 
391  // We don't modify the types of PHIs. Reductions will already have been
392  // truncated if possible, and inductions' sizes will have been chosen by
393  // indvars.
394  if (isa<PHINode>(I))
395  continue;
396 
397  if (DBits[Leader] == ~0ULL)
398  // All bits demanded, no point continuing.
399  continue;
400 
401  for (Value *O : cast<User>(I)->operands()) {
402  ECs.unionSets(Leader, O);
403  Worklist.push_back(O);
404  }
405  }
406 
407  // Now we've discovered all values, walk them to see if there are
408  // any users we didn't see. If there are, we can't optimize that
409  // chain.
410  for (auto &I : DBits)
411  for (auto *U : I.first->users())
412  if (U->getType()->isIntegerTy() && DBits.count(U) == 0)
413  DBits[ECs.getOrInsertLeaderValue(I.first)] |= ~0ULL;
414 
415  for (auto I = ECs.begin(), E = ECs.end(); I != E; ++I) {
416  uint64_t LeaderDemandedBits = 0;
417  for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI)
418  LeaderDemandedBits |= DBits[*MI];
419 
420  uint64_t MinBW = (sizeof(LeaderDemandedBits) * 8) -
421  llvm::countLeadingZeros(LeaderDemandedBits);
422  // Round up to a power of 2
423  if (!isPowerOf2_64((uint64_t)MinBW))
424  MinBW = NextPowerOf2(MinBW);
425 
426  // We don't modify the types of PHIs. Reductions will already have been
427  // truncated if possible, and inductions' sizes will have been chosen by
428  // indvars.
429  // If we are required to shrink a PHI, abandon this entire equivalence class.
430  bool Abort = false;
431  for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI)
432  if (isa<PHINode>(*MI) && MinBW < (*MI)->getType()->getScalarSizeInBits()) {
433  Abort = true;
434  break;
435  }
436  if (Abort)
437  continue;
438 
439  for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI) {
440  if (!isa<Instruction>(*MI))
441  continue;
442  Type *Ty = (*MI)->getType();
443  if (Roots.count(*MI))
444  Ty = cast<Instruction>(*MI)->getOperand(0)->getType();
445  if (MinBW < Ty->getScalarSizeInBits())
446  MinBWs[cast<Instruction>(*MI)] = MinBW;
447  }
448  }
449 
450  return MinBWs;
451 }
452 
453 /// \returns \p I after propagating metadata from \p VL.
455  Instruction *I0 = cast<Instruction>(VL[0]);
457  I0->getAllMetadataOtherThanDebugLoc(Metadata);
458 
459  for (auto Kind :
463  MDNode *MD = I0->getMetadata(Kind);
464 
465  for (int J = 1, E = VL.size(); MD && J != E; ++J) {
466  const Instruction *IJ = cast<Instruction>(VL[J]);
467  MDNode *IMD = IJ->getMetadata(Kind);
468  switch (Kind) {
470  MD = MDNode::getMostGenericTBAA(MD, IMD);
471  break;
473  MD = MDNode::getMostGenericAliasScope(MD, IMD);
474  break;
476  MD = MDNode::getMostGenericFPMath(MD, IMD);
477  break;
481  MD = MDNode::intersect(MD, IMD);
482  break;
483  default:
484  llvm_unreachable("unhandled metadata");
485  }
486  }
487 
488  Inst->setMetadata(Kind, MD);
489  }
490 
491  return Inst;
492 }
493 
495  unsigned NumVecs) {
497  for (unsigned i = 0; i < VF; i++)
498  for (unsigned j = 0; j < NumVecs; j++)
499  Mask.push_back(Builder.getInt32(j * VF + i));
500 
501  return ConstantVector::get(Mask);
502 }
503 
504 Constant *llvm::createStrideMask(IRBuilder<> &Builder, unsigned Start,
505  unsigned Stride, unsigned VF) {
507  for (unsigned i = 0; i < VF; i++)
508  Mask.push_back(Builder.getInt32(Start + i * Stride));
509 
510  return ConstantVector::get(Mask);
511 }
512 
514  unsigned NumInts, unsigned NumUndefs) {
516  for (unsigned i = 0; i < NumInts; i++)
517  Mask.push_back(Builder.getInt32(Start + i));
518 
520  for (unsigned i = 0; i < NumUndefs; i++)
521  Mask.push_back(Undef);
522 
523  return ConstantVector::get(Mask);
524 }
525 
526 /// A helper function for concatenating vectors. This function concatenates two
527 /// vectors having the same element type. If the second vector has fewer
528 /// elements than the first, it is padded with undefs.
530  Value *V2) {
531  VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType());
532  VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());
533  assert(VecTy1 && VecTy2 &&
534  VecTy1->getScalarType() == VecTy2->getScalarType() &&
535  "Expect two vectors with the same element type");
536 
537  unsigned NumElts1 = VecTy1->getNumElements();
538  unsigned NumElts2 = VecTy2->getNumElements();
539  assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements");
540 
541  if (NumElts1 > NumElts2) {
542  // Extend with UNDEFs.
543  Constant *ExtMask =
544  createSequentialMask(Builder, 0, NumElts2, NumElts1 - NumElts2);
545  V2 = Builder.CreateShuffleVector(V2, UndefValue::get(VecTy2), ExtMask);
546  }
547 
548  Constant *Mask = createSequentialMask(Builder, 0, NumElts1 + NumElts2, 0);
549  return Builder.CreateShuffleVector(V1, V2, Mask);
550 }
551 
553  unsigned NumVecs = Vecs.size();
554  assert(NumVecs > 1 && "Should be at least two vectors");
555 
556  SmallVector<Value *, 8> ResList;
557  ResList.append(Vecs.begin(), Vecs.end());
558  do {
559  SmallVector<Value *, 8> TmpList;
560  for (unsigned i = 0; i < NumVecs - 1; i += 2) {
561  Value *V0 = ResList[i], *V1 = ResList[i + 1];
562  assert((V0->getType() == V1->getType() || i == NumVecs - 2) &&
563  "Only the last vector may have a different type");
564 
565  TmpList.push_back(concatenateTwoVectors(Builder, V0, V1));
566  }
567 
568  // Push the last vector if the total number of vectors is odd.
569  if (NumVecs % 2 != 0)
570  TmpList.push_back(ResList[NumVecs - 1]);
571 
572  ResList = TmpList;
573  NumVecs = ResList.size();
574  } while (NumVecs > 1);
575 
576  return ResList[0];
577 }
void getAllMetadataOtherThanDebugLoc(SmallVectorImpl< std::pair< unsigned, MDNode *>> &MDs) const
This does the same thing as getAllMetadata, except that it filters out the debug location.
Definition: Instruction.h:217
Value * getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
Get the stride of a pointer access in a loop.
uint64_t CallInst * C
Value * stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
If the argument is a GEP, then returns the operand identified by getGEPInductionOperand.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:109
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:72
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1542
MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock *> Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register...
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
iterator begin() const
Definition: ArrayRef.h:137
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
match_zero m_Zero()
Match an arbitrary zero/null constant.
Definition: PatternMatch.h:145
const Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value *> VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal].
The main scalar evolution driver.
static MDNode * getMostGenericAliasScope(MDNode *A, MDNode *B)
Definition: Metadata.cpp:915
This class represents a function call, abstracting a target machine&#39;s calling convention.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:91
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
This instruction constructs a fixed permutation of two input vectors.
This class implements a map that also provides access to all stored values in a deterministic order...
Definition: MapVector.h:38
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
Metadata node.
Definition: Metadata.h:862
member_iterator unionSets(const ElemTy &V1, const ElemTy &V2)
union - Merge the two equivalence sets for the specified values, inserting them if they do not alread...
Hexagon Common GEP
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:227
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:181
This is the base class for unary cast operator classes.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1488
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
Definition: VectorUtils.cpp:87
Intrinsic::ID getIntrinsicForCallSite(ImmutableCallSite ICS, const TargetLibraryInfo *TLI)
Map a call instruction to an intrinsic ID.
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:348
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:361
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:560
static Value * concatenateTwoVectors(IRBuilder<> &Builder, Value *V1, Value *V2)
A helper function for concatenating vectors.
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:197
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:668
uint64_t getNumElements() const
Definition: DerivedTypes.h:359
This node represents multiplication of some number of SCEVs.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
Definition: PatternMatch.h:502
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1554
Constant * createSequentialMask(IRBuilder<> &Builder, unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
member_iterator member_begin(iterator I) const
This node represents a polynomial recurrence on the trip count of the specified loop.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:194
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static MDNode * intersect(MDNode *A, MDNode *B)
Definition: Metadata.cpp:902
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
Value * getOperand(unsigned i) const
Definition: User.h:154
Class to represent pointers.
Definition: DerivedTypes.h:467
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:277
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:301
bool isTypeLegal(Type *Ty) const
Return true if this type is legal.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
Definition: Instructions.h:837
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This instruction inserts a single (scalar) element into a VectorType value.
Value * concatenateVectors(IRBuilder<> &Builder, ArrayRef< Value *> Vecs)
Concatenate a list of vectors.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:149
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
member_iterator member_end() const
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:426
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
static double log2(double V)
static MDNode * getMostGenericTBAA(MDNode *A, MDNode *B)
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1320
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:632
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1214
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1238
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:56
unsigned getNumOperands() const
Definition: User.h:176
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:418
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
const ElemTy & getOrInsertLeaderValue(const ElemTy &V)
getOrInsertLeaderValue - Return the leader for the specified value that is in the set...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
Provides information about what library functions are available for the current target.
iterator end() const
Definition: ArrayRef.h:138
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:385
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:308
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition: IRBuilder.h:1740
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:72
APInt getDemandedBits(Instruction *I)
Return the bits demanded from instruction I.
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:462
Class to represent vector types.
Definition: DerivedTypes.h:393
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:57
Class for arbitrary precision integers.
Definition: APInt.h:69
iterator_range< user_iterator > users()
Definition: Value.h:401
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:398
Value * getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty)
If a value has only one user that is a CastInst, return it.
Constant * createStrideMask(IRBuilder<> &Builder, unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
unsigned getGEPInductionOperand(const GetElementPtrInst *Gep)
Find the operand of the GEP that should be checked for consecutive stores.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:403
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:538
This class represents an analyzed expression in the program.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:439
#define I(x, y, z)
Definition: MD5.cpp:58
Type * getResultElementType() const
Definition: Instructions.h:939
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
const unsigned Kind
Constant * createInterleaveMask(IRBuilder<> &Builder, unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:73
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
IRTranslator LLVM IR MI
This pass exposes codegen information to IR-level passes.
static MDNode * getMostGenericFPMath(MDNode *A, MDNode *B)
Definition: Metadata.cpp:922
static Constant * get(ArrayRef< Constant *> V)
Definition: Constants.cpp:984
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1227
std::vector< uint32_t > Metadata
PAL metadata represented as a vector.
gep_type_iterator gep_type_begin(const User *GEP)
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:35