LLVM  3.7.0
VectorUtils.cpp
Go to the documentation of this file.
1 //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines vectorizer utilities.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Analysis/LoopInfo.h"
19 #include "llvm/IR/PatternMatch.h"
20 #include "llvm/IR/Value.h"
21 
22 /// \brief Identify if the intrinsic is trivially vectorizable.
23 /// This method returns true if the intrinsic's argument types are all
24 /// scalars for the scalar form of the intrinsic and all vectors for
25 /// the vector form of the intrinsic.
27  switch (ID) {
28  case Intrinsic::sqrt:
29  case Intrinsic::sin:
30  case Intrinsic::cos:
31  case Intrinsic::exp:
32  case Intrinsic::exp2:
33  case Intrinsic::log:
34  case Intrinsic::log10:
35  case Intrinsic::log2:
36  case Intrinsic::fabs:
37  case Intrinsic::minnum:
38  case Intrinsic::maxnum:
39  case Intrinsic::copysign:
40  case Intrinsic::floor:
41  case Intrinsic::ceil:
42  case Intrinsic::trunc:
43  case Intrinsic::rint:
44  case Intrinsic::nearbyint:
45  case Intrinsic::round:
46  case Intrinsic::bswap:
47  case Intrinsic::ctpop:
48  case Intrinsic::pow:
49  case Intrinsic::fma:
50  case Intrinsic::fmuladd:
51  case Intrinsic::ctlz:
52  case Intrinsic::cttz:
53  case Intrinsic::powi:
54  return true;
55  default:
56  return false;
57  }
58 }
59 
60 /// \brief Identifies if the intrinsic has a scalar operand. It check for
61 /// ctlz,cttz and powi special intrinsics whose argument is scalar.
63  unsigned ScalarOpdIdx) {
64  switch (ID) {
65  case Intrinsic::ctlz:
66  case Intrinsic::cttz:
67  case Intrinsic::powi:
68  return (ScalarOpdIdx == 1);
69  default:
70  return false;
71  }
72 }
73 
74 /// \brief Check call has a unary float signature
75 /// It checks following:
76 /// a) call should have a single argument
77 /// b) argument type should be floating point type
78 /// c) call instruction type and argument type should be same
79 /// d) call should only reads memory.
80 /// If all these condition is met then return ValidIntrinsicID
81 /// else return not_intrinsic.
84  Intrinsic::ID ValidIntrinsicID) {
85  if (I.getNumArgOperands() != 1 ||
87  I.getType() != I.getArgOperand(0)->getType() || !I.onlyReadsMemory())
89 
90  return ValidIntrinsicID;
91 }
92 
93 /// \brief Check call has a binary float signature
94 /// It checks following:
95 /// a) call should have 2 arguments.
96 /// b) arguments type should be floating point type
97 /// c) call instruction type and arguments type should be same
98 /// d) call should only reads memory.
99 /// If all these condition is met then return ValidIntrinsicID
100 /// else return not_intrinsic.
103  Intrinsic::ID ValidIntrinsicID) {
104  if (I.getNumArgOperands() != 2 ||
107  I.getType() != I.getArgOperand(0)->getType() ||
108  I.getType() != I.getArgOperand(1)->getType() || !I.onlyReadsMemory())
110 
111  return ValidIntrinsicID;
112 }
113 
114 /// \brief Returns intrinsic ID for call.
115 /// For the input call instruction it finds mapping intrinsic and returns
116 /// its ID, in case it does not found it return not_intrinsic.
118  const TargetLibraryInfo *TLI) {
119  // If we have an intrinsic call, check if it is trivially vectorizable.
120  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
121  Intrinsic::ID ID = II->getIntrinsicID();
122  if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
123  ID == Intrinsic::lifetime_end || ID == Intrinsic::assume)
124  return ID;
126  }
127 
128  if (!TLI)
130 
132  Function *F = CI->getCalledFunction();
133  // We're going to make assumptions on the semantics of the functions, check
134  // that the target knows that it's available in this environment and it does
135  // not have local linkage.
136  if (!F || F->hasLocalLinkage() || !TLI->getLibFunc(F->getName(), Func))
138 
139  // Otherwise check if we have a call to a function that can be turned into a
140  // vector intrinsic.
141  switch (Func) {
142  default:
143  break;
144  case LibFunc::sin:
145  case LibFunc::sinf:
146  case LibFunc::sinl:
147  return checkUnaryFloatSignature(*CI, Intrinsic::sin);
148  case LibFunc::cos:
149  case LibFunc::cosf:
150  case LibFunc::cosl:
151  return checkUnaryFloatSignature(*CI, Intrinsic::cos);
152  case LibFunc::exp:
153  case LibFunc::expf:
154  case LibFunc::expl:
155  return checkUnaryFloatSignature(*CI, Intrinsic::exp);
156  case LibFunc::exp2:
157  case LibFunc::exp2f:
158  case LibFunc::exp2l:
159  return checkUnaryFloatSignature(*CI, Intrinsic::exp2);
160  case LibFunc::log:
161  case LibFunc::logf:
162  case LibFunc::logl:
163  return checkUnaryFloatSignature(*CI, Intrinsic::log);
164  case LibFunc::log10:
165  case LibFunc::log10f:
166  case LibFunc::log10l:
167  return checkUnaryFloatSignature(*CI, Intrinsic::log10);
168  case LibFunc::log2:
169  case LibFunc::log2f:
170  case LibFunc::log2l:
171  return checkUnaryFloatSignature(*CI, Intrinsic::log2);
172  case LibFunc::fabs:
173  case LibFunc::fabsf:
174  case LibFunc::fabsl:
175  return checkUnaryFloatSignature(*CI, Intrinsic::fabs);
176  case LibFunc::fmin:
177  case LibFunc::fminf:
178  case LibFunc::fminl:
180  case LibFunc::fmax:
181  case LibFunc::fmaxf:
182  case LibFunc::fmaxl:
184  case LibFunc::copysign:
185  case LibFunc::copysignf:
186  case LibFunc::copysignl:
187  return checkBinaryFloatSignature(*CI, Intrinsic::copysign);
188  case LibFunc::floor:
189  case LibFunc::floorf:
190  case LibFunc::floorl:
191  return checkUnaryFloatSignature(*CI, Intrinsic::floor);
192  case LibFunc::ceil:
193  case LibFunc::ceilf:
194  case LibFunc::ceill:
195  return checkUnaryFloatSignature(*CI, Intrinsic::ceil);
196  case LibFunc::trunc:
197  case LibFunc::truncf:
198  case LibFunc::truncl:
199  return checkUnaryFloatSignature(*CI, Intrinsic::trunc);
200  case LibFunc::rint:
201  case LibFunc::rintf:
202  case LibFunc::rintl:
203  return checkUnaryFloatSignature(*CI, Intrinsic::rint);
204  case LibFunc::nearbyint:
205  case LibFunc::nearbyintf:
206  case LibFunc::nearbyintl:
207  return checkUnaryFloatSignature(*CI, Intrinsic::nearbyint);
208  case LibFunc::round:
209  case LibFunc::roundf:
210  case LibFunc::roundl:
211  return checkUnaryFloatSignature(*CI, Intrinsic::round);
212  case LibFunc::pow:
213  case LibFunc::powf:
214  case LibFunc::powl:
215  return checkBinaryFloatSignature(*CI, Intrinsic::pow);
216  }
217 
219 }
220 
221 /// \brief Find the operand of the GEP that should be checked for consecutive
222 /// stores. This ignores trailing indices that have no effect on the final
223 /// pointer.
225  const DataLayout &DL = Gep->getModule()->getDataLayout();
226  unsigned LastOperand = Gep->getNumOperands() - 1;
227  unsigned GEPAllocSize = DL.getTypeAllocSize(
228  cast<PointerType>(Gep->getType()->getScalarType())->getElementType());
229 
230  // Walk backwards and try to peel off zeros.
231  while (LastOperand > 1 &&
232  match(Gep->getOperand(LastOperand), llvm::PatternMatch::m_Zero())) {
233  // Find the type we're currently indexing into.
234  gep_type_iterator GEPTI = gep_type_begin(Gep);
235  std::advance(GEPTI, LastOperand - 1);
236 
237  // If it's a type with the same allocation size as the result of the GEP we
238  // can peel off the zero index.
239  if (DL.getTypeAllocSize(*GEPTI) != GEPAllocSize)
240  break;
241  --LastOperand;
242  }
243 
244  return LastOperand;
245 }
246 
247 /// \brief If the argument is a GEP, then returns the operand identified by
248 /// getGEPInductionOperand. However, if there is some other non-loop-invariant
249 /// operand, it returns that instead.
251  Loop *Lp) {
253  if (!GEP)
254  return Ptr;
255 
256  unsigned InductionOperand = getGEPInductionOperand(GEP);
257 
258  // Check that all of the gep indices are uniform except for our induction
259  // operand.
260  for (unsigned i = 0, e = GEP->getNumOperands(); i != e; ++i)
261  if (i != InductionOperand &&
262  !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(i)), Lp))
263  return Ptr;
264  return GEP->getOperand(InductionOperand);
265 }
266 
267 /// \brief If a value has only one user that is a CastInst, return it.
269  llvm::Value *UniqueCast = nullptr;
270  for (User *U : Ptr->users()) {
271  CastInst *CI = dyn_cast<CastInst>(U);
272  if (CI && CI->getType() == Ty) {
273  if (!UniqueCast)
274  UniqueCast = CI;
275  else
276  return nullptr;
277  }
278  }
279  return UniqueCast;
280 }
281 
282 /// \brief Get the stride of a pointer access in a loop. Looks for symbolic
283 /// strides "a[i*stride]". Returns the symbolic stride, or null otherwise.
285  Loop *Lp) {
286  const PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
287  if (!PtrTy || PtrTy->isAggregateType())
288  return nullptr;
289 
290  // Try to remove a gep instruction to make the pointer (actually index at this
291  // point) easier analyzable. If OrigPtr is equal to Ptr we are analzying the
292  // pointer, otherwise, we are analyzing the index.
293  llvm::Value *OrigPtr = Ptr;
294 
295  // The size of the pointer access.
296  int64_t PtrAccessSize = 1;
297 
298  Ptr = stripGetElementPtr(Ptr, SE, Lp);
299  const SCEV *V = SE->getSCEV(Ptr);
300 
301  if (Ptr != OrigPtr)
302  // Strip off casts.
303  while (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V))
304  V = C->getOperand();
305 
306  const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V);
307  if (!S)
308  return nullptr;
309 
310  V = S->getStepRecurrence(*SE);
311  if (!V)
312  return nullptr;
313 
314  // Strip off the size of access multiplication if we are still analyzing the
315  // pointer.
316  if (OrigPtr == Ptr) {
317  const DataLayout &DL = Lp->getHeader()->getModule()->getDataLayout();
318  DL.getTypeAllocSize(PtrTy->getElementType());
319  if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
320  if (M->getOperand(0)->getSCEVType() != scConstant)
321  return nullptr;
322 
323  const APInt &APStepVal =
324  cast<SCEVConstant>(M->getOperand(0))->getValue()->getValue();
325 
326  // Huge step value - give up.
327  if (APStepVal.getBitWidth() > 64)
328  return nullptr;
329 
330  int64_t StepVal = APStepVal.getSExtValue();
331  if (PtrAccessSize != StepVal)
332  return nullptr;
333  V = M->getOperand(1);
334  }
335  }
336 
337  // Strip off casts.
338  Type *StripedOffRecurrenceCast = nullptr;
339  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V)) {
340  StripedOffRecurrenceCast = C->getType();
341  V = C->getOperand();
342  }
343 
344  // Look for the loop invariant symbolic value.
345  const SCEVUnknown *U = dyn_cast<SCEVUnknown>(V);
346  if (!U)
347  return nullptr;
348 
349  llvm::Value *Stride = U->getValue();
350  if (!Lp->isLoopInvariant(Stride))
351  return nullptr;
352 
353  // If we have stripped off the recurrence cast we have to make sure that we
354  // return the value that is used in this loop so that we can replace it later.
355  if (StripedOffRecurrenceCast)
356  Stride = getUniqueCastUse(Stride, Lp, StripedOffRecurrenceCast);
357 
358  return Stride;
359 }
360 
361 /// \brief Given a vector and an element number, see if the scalar value is
362 /// already around as a register, for example if it were inserted then extracted
363 /// from the vector.
365  assert(V->getType()->isVectorTy() && "Not looking at a vector?");
366  VectorType *VTy = cast<VectorType>(V->getType());
367  unsigned Width = VTy->getNumElements();
368  if (EltNo >= Width) // Out of range access.
369  return UndefValue::get(VTy->getElementType());
370 
371  if (Constant *C = dyn_cast<Constant>(V))
372  return C->getAggregateElement(EltNo);
373 
374  if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) {
375  // If this is an insert to a variable element, we don't know what it is.
376  if (!isa<ConstantInt>(III->getOperand(2)))
377  return nullptr;
378  unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
379 
380  // If this is an insert to the element we are looking for, return the
381  // inserted value.
382  if (EltNo == IIElt)
383  return III->getOperand(1);
384 
385  // Otherwise, the insertelement doesn't modify the value, recurse on its
386  // vector input.
387  return findScalarElement(III->getOperand(0), EltNo);
388  }
389 
390  if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) {
391  unsigned LHSWidth = SVI->getOperand(0)->getType()->getVectorNumElements();
392  int InEl = SVI->getMaskValue(EltNo);
393  if (InEl < 0)
394  return UndefValue::get(VTy->getElementType());
395  if (InEl < (int)LHSWidth)
396  return findScalarElement(SVI->getOperand(0), InEl);
397  return findScalarElement(SVI->getOperand(1), InEl - LHSWidth);
398  }
399 
400  // Extract a value from a vector add operation with a constant zero.
401  Value *Val = nullptr; Constant *Con = nullptr;
402  if (match(V,
405  if (Constant *Elt = Con->getAggregateElement(EltNo))
406  if (Elt->isNullValue())
407  return findScalarElement(Val, EltNo);
408  }
409 
410  // Otherwise, we don't know.
411  return nullptr;
412 }
Intrinsic::ID getIntrinsicIDForCall(CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:64
Value * stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
If the argument is a GEP, then returns the operand identified by getGEPInductionOperand.
match_zero m_Zero()
Match an arbitrary zero/null constant.
Definition: PatternMatch.h:137
unsigned getNumOperands() const
Definition: User.h:138
ScalarEvolution - This class is the main scalar evolution driver.
CallInst - This class represents a function call, abstracting a target machine's calling convention...
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:83
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
getStepRecurrence - This method constructs and returns the recurrence indicating how much this expres...
ShuffleVectorInst - This instruction constructs a fixed permutation of two input vectors.
bool isLoopInvariant(const SCEV *S, const Loop *L)
isLoopInvariant - Return true if the value of the given SCEV is unchanging in the specified loop...
F(f)
Hexagon Common GEP
BlockT * getHeader() const
Definition: LoopInfo.h:96
SCEVCastExpr - This is the base class for unary cast operator classes.
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:41
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:389
Value * getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty)
If a value has only one user that is a CastInst, return it.
static void advance(T &it, size_t Val)
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
bool isLoopInvariant(const Value *V) const
isLoopInvariant - Return true if the specified value is loop invariant
Definition: LoopInfo.cpp:59
SCEVMulExpr - This node represents multiplication of some number of SCEVs.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
Definition: PatternMatch.h:434
Value * getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
Get the stride of a pointer access in a loop.
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
bool isFloatingPointTy() const
isFloatingPointTy - Return true if this is one of the six floating point types
Definition: Type.h:159
unsigned getNumElements() const
Return the number of elements in the Vector type.
Definition: DerivedTypes.h:432
Type * getElementType() const
Definition: DerivedTypes.h:323
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register...
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
Definition: Instructions.h:830
bool getLibFunc(StringRef funcName, LibFunc::Func &F) const
Searches for a particular function name.
SCEVUnknown - This means that we are dealing with an entirely unknown SCEV value, and only represent ...
InsertElementInst - This instruction inserts a single (scalar) element into a VectorType value...
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
This is an important base class in LLVM.
Definition: Constant.h:41
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1339
Intrinsic::ID checkUnaryFloatSignature(const CallInst &I, Intrinsic::ID ValidIntrinsicID)
Identify if call has a unary float signature It returns input intrinsic ID if call has a single argum...
Definition: VectorUtils.cpp:83
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1273
Value * getOperand(unsigned i) const
Definition: User.h:118
Constant * getAggregateElement(unsigned Elt) const
getAggregateElement - For aggregates (struct/array/vector) return the constant that corresponds to th...
Definition: Constants.cpp:250
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:670
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:388
unsigned getVectorNumElements() const
Definition: Type.cpp:212
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:57
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
Provides information about what library functions are available for the current target.
SequentialType * getType() const
Definition: Instructions.h:922
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:62
Intrinsic::ID checkBinaryFloatSignature(const CallInst &I, Intrinsic::ID ValidIntrinsicID)
Identify if call has a binary float signature It returns input intrinsic ID if call has two arguments...
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
VectorType - Class to represent vector types.
Definition: DerivedTypes.h:362
Class for arbitrary precision integers.
Definition: APInt.h:73
iterator_range< user_iterator > users()
Definition: Value.h:300
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
const Type * getScalarType() const LLVM_READONLY
getScalarType - If this is a vector type, return the element type, otherwise return 'this'...
Definition: Type.cpp:51
unsigned getGEPInductionOperand(const GetElementPtrInst *Gep)
Find the operand of the GEP that should be checked for consecutive stores.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:372
bool isAggregateType() const
isAggregateType - Return true if the type is an aggregate type.
Definition: Type.h:260
SCEV - This class represents an analyzed expression in the program.
#define I(x, y, z)
Definition: MD5.cpp:54
bool hasLocalLinkage() const
Definition: GlobalValue.h:280
LLVM Value Representation.
Definition: Value.h:69
const SCEV * getSCEV(Value *V)
getSCEV - Return a SCEV expression for the full generality of the specified expression.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:659
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:37
gep_type_iterator gep_type_begin(const User *GEP)
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:26