LLVM  4.0.0
Classes | Namespaces | Macros | Functions | Variables
SLPVectorizer.cpp File Reference
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NoFolder.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Vectorize.h"
#include <algorithm>
#include <memory>
Include dependency graph for SLPVectorizer.cpp:

Go to the source code of this file.

Classes

class  llvm::slpvectorizer::BoUpSLP
 Bottom Up SLP Vectorizer. More...
 
struct  llvm::slpvectorizer::BoUpSLP::BlockScheduling::ReadyList
 

Namespaces

 llvm
 Compute iterated dominance frontiers using a linear time algorithm.
 
 llvm::slpvectorizer
 A private "module" namespace for types and utilities used by this pass.
 

Macros

#define SV_NAME   "slp-vectorizer"
 
#define DEBUG_TYPE   "SLP"
 

Functions

 STATISTIC (NumVectorInstructions,"Number of vector instructions generated")
 
static bool isValidElementType (Type *Ty)
 Predicate for the element types that the SLP vectorizer supports. More...
 
static bool allSameBlock (ArrayRef< Value * > VL)
 
static bool allConstant (ArrayRef< Value * > VL)
 
static bool isSplat (ArrayRef< Value * > VL)
 
static unsigned getAltOpcode (unsigned Op)
 
static bool canCombineAsAltInst (unsigned Op)
 
static unsigned isAltInst (ArrayRef< Value * > VL)
 
static unsigned getSameOpcode (ArrayRef< Value * > VL)
 
static void propagateIRFlags (Value *I, ArrayRef< Value * > VL)
 Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) that will be converted into a vector (I). More...
 
static bool allSameType (ArrayRef< Value * > VL)
 
static bool matchExtractIndex (Instruction *E, unsigned Idx, unsigned Opcode)
 
static bool InTreeUserNeedToExtract (Value *Scalar, Instruction *UserInst, TargetLibraryInfo *TLI)
 
static MemoryLocation getLocation (Instruction *I, AliasAnalysis *AA)
 
static bool isSimple (Instruction *I)
 
static bool shouldReorderOperands (int i, Instruction &I, SmallVectorImpl< Value * > &Left, SmallVectorImpl< Value * > &Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight, bool SplatLeft, bool SplatRight)
 
static bool collectValuesToDemote (Value *V, SmallPtrSetImpl< Value * > &Expr, SmallVectorImpl< Value * > &ToDemote, SmallVectorImpl< Value * > &Roots)
 
static bool hasValueBeenRAUWed (ArrayRef< Value * > VL, ArrayRef< WeakVH > VH, unsigned SliceBegin, unsigned SliceSize)
 Check that the Values in the slice in VL array are still existent in the WeakVH array. More...
 
static ValuecreateRdxShuffleMask (unsigned VecLen, unsigned NumEltsToRdx, bool IsPairwise, bool IsLeft, IRBuilder<> &Builder)
 Generate a shuffle mask to be used in a reduction tree. More...
 
static bool findBuildVector (InsertElementInst *FirstInsertElem, SmallVectorImpl< Value * > &BuildVector, SmallVectorImpl< Value * > &BuildVectorOpds)
 Recognize construction of vectors like ra = insertelement <4 x float> undef, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3. More...
 
static bool findBuildAggregate (InsertValueInst *IV, SmallVectorImpl< Value * > &BuildVector, SmallVectorImpl< Value * > &BuildVectorOpds)
 Like findBuildVector, but looks backwards for construction of aggregate. More...
 
static bool PhiTypeSorterFunc (Value *V, Value *V2)
 
static ValuegetReductionValue (const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI)
 Try and get a reduction value from a phi node. More...
 
static bool canMatchHorizontalReduction (PHINode *P, BinaryOperator *BI, BoUpSLP &R, TargetTransformInfo *TTI, unsigned MinRegSize)
 Attempt to reduce a horizontal reduction. More...
 
Passllvm::createSLPVectorizerPass ()
 

Variables

static cl::opt< int > SLPCostThreshold ("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this ""number "))
 
static cl::opt< boolShouldVectorizeHor ("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions"))
 
static cl::opt< boolShouldStartVectorizeHorAtStore ("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store"))
 
static cl::opt< int > MaxVectorRegSizeOption ("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
 
static cl::opt< int > ScheduleRegionSizeBudget ("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block"))
 Limits the size of scheduling regions in a block. More...
 
static cl::opt< int > MinVectorRegSizeOption ("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
 
static cl::opt< unsignedRecursionMaxDepth ("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree"))
 
static cl::opt< unsignedMinTreeSize ("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable"))
 
static const unsigned AliasedCheckLimit = 10
 
static const unsigned MaxMemDepDistance = 160
 
static const int MinScheduleRegionSize = 16
 If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled. More...
 
static const char lv_name [] = "SLP Vectorizer"
 

Macro Definition Documentation

#define DEBUG_TYPE   "SLP"

Definition at line 51 of file SLPVectorizer.cpp.

#define SV_NAME   "slp-vectorizer"

Definition at line 50 of file SLPVectorizer.cpp.

Function Documentation

static bool allConstant ( ArrayRef< Value * >  VL)
static
Returns
True if all of the values in VL are constants.

Definition at line 137 of file SLPVectorizer.cpp.

References i.

static bool allSameBlock ( ArrayRef< Value * >  VL)
static
Returns
true if all of the instructions in VL are in the same block or false otherwise.

Definition at line 120 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::Instruction::getParent(), I, i, and llvm::ArrayRef< T >::size().

static bool allSameType ( ArrayRef< Value * >  VL)
static
Returns
true if all of the values in VL have the same type or false otherwise.

Definition at line 229 of file SLPVectorizer.cpp.

References llvm::Intrinsic::getType(), i, and llvm::ArrayRef< T >::size().

Referenced by llvm::slpvectorizer::BoUpSLP::buildTree().

static bool canCombineAsAltInst ( unsigned  Op)
static
Returns
bool representing if Opcode Op can be part of an alternate sequence which can later be merged as a ShuffleVector instruction.

Definition at line 172 of file SLPVectorizer.cpp.

References llvm::MCID::Add.

Referenced by getSameOpcode().

static bool canMatchHorizontalReduction ( PHINode P,
BinaryOperator BI,
BoUpSLP R,
TargetTransformInfo TTI,
unsigned  MinRegSize 
)
static

Attempt to reduce a horizontal reduction.

If it is legal to match a horizontal reduction feeding the phi node P with reduction operators BI, then check if it can be done.

Returns
true if a horizontal reduction was matched and reduced.
false if a horizontal reduction was not matched.

Definition at line 4516 of file SLPVectorizer.cpp.

References llvm::PowerOf2Floor(), and ShouldVectorizeHor.

static bool collectValuesToDemote ( Value V,
SmallPtrSetImpl< Value * > &  Expr,
SmallVectorImpl< Value * > &  ToDemote,
SmallVectorImpl< Value * > &  Roots 
)
static
static Value* createRdxShuffleMask ( unsigned  VecLen,
unsigned  NumEltsToRdx,
bool  IsPairwise,
bool  IsLeft,
IRBuilder<> &  Builder 
)
static

Generate a shuffle mask to be used in a reduction tree.

Parameters
VecLenThe length of the vector to be reduced.
NumEltsToRdxThe number of elements that should be reduced in the vector.
IsPairwiseWhether the reduction is a pairwise or splitting reduction. A pairwise reduction will generate a mask of <0,2,...> or <1,3,..> while a splitting reduction will generate <2,3, undef,undef> for a vector of 4 and NumElts = 2.
IsLeftTrue will generate a mask of even elements, odd otherwise.

Definition at line 4071 of file SLPVectorizer.cpp.

References assert(), llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::IRBuilderBase::getInt32(), llvm::IRBuilderBase::getInt32Ty(), and i.

static bool findBuildAggregate ( InsertValueInst IV,
SmallVectorImpl< Value * > &  BuildVector,
SmallVectorImpl< Value * > &  BuildVectorOpds 
)
static

Like findBuildVector, but looks backwards for construction of aggregate.

Returns
true if it matches.

Definition at line 4438 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::InsertValueInst::getAggregateOperand(), llvm::InsertValueInst::getInsertedValueOperand(), llvm::Value::hasOneUse(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().

static bool findBuildVector ( InsertElementInst FirstInsertElem,
SmallVectorImpl< Value * > &  BuildVector,
SmallVectorImpl< Value * > &  BuildVectorOpds 
)
static

Recognize construction of vectors like ra = insertelement <4 x float> undef, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3.

Returns true if it matches

Definition at line 4406 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::User::getOperand(), llvm::Value::hasOneUse(), llvm::ARM_PROC::IE, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::Value::use_empty(), and llvm::Instruction::user_back().

static unsigned getAltOpcode ( unsigned  Op)
static
Returns
Opcode that can be clubbed with Op to create an alternate sequence which can later be merged as a ShuffleVector instruction.

Definition at line 154 of file SLPVectorizer.cpp.

References llvm::MCID::Add.

Referenced by isAltInst().

static MemoryLocation getLocation ( Instruction I,
AliasAnalysis AA 
)
static
Returns
the AA location that is being access by the instruction.

Definition at line 279 of file SLPVectorizer.cpp.

References llvm::MemoryLocation::get(), and SI.

static Value* getReductionValue ( const DominatorTree DT,
PHINode P,
BasicBlock ParentBB,
LoopInfo LI 
)
static

Try and get a reduction value from a phi node.

Given a phi node P in a block ParentBB, consider possible reductions if they come from either ParentBB or a containing loop latch.

Returns
A candidate reduction value if possible, or
nullptr
if not possible.

Definition at line 4465 of file SLPVectorizer.cpp.

References llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::LoopInfoBase< N, M >::getLoopFor(), llvm::LoopBase< N, M >::getLoopLatch(), llvm::Instruction::getParent(), and getParent().

static unsigned getSameOpcode ( ArrayRef< Value * >  VL)
static
Returns
The opcode if all of the Instructions in VL have the same opcode, or zero.

Definition at line 194 of file SLPVectorizer.cpp.

References canCombineAsAltInst(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), I, i, isAltInst(), and llvm::ArrayRef< T >::size().

static bool hasValueBeenRAUWed ( ArrayRef< Value * >  VL,
ArrayRef< WeakVH VH,
unsigned  SliceBegin,
unsigned  SliceSize 
)
static

Check that the Values in the slice in VL array are still existent in the WeakVH array.

Vectorization of part of the VL array may cause later values in the VL array to become invalid. We track when this has happened in the WeakVH array.

Definition at line 3729 of file SLPVectorizer.cpp.

References llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::lltok::equal, and llvm::ArrayRef< T >::slice().

static bool InTreeUserNeedToExtract ( Value Scalar,
Instruction UserInst,
TargetLibraryInfo TLI 
)
static
static unsigned isAltInst ( ArrayRef< Value * >  VL)
static
Returns
ShuffleVector instruction if instructions in VL have alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence. (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...)

Definition at line 180 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), getAltOpcode(), llvm::Instruction::getOpcode(), I, i, and llvm::ArrayRef< T >::size().

Referenced by getSameOpcode().

static bool isSimple ( Instruction I)
static
static bool isSplat ( ArrayRef< Value * >  VL)
static
Returns
True if all of the values in VL are identical.

Definition at line 145 of file SLPVectorizer.cpp.

References i, and llvm::ArrayRef< T >::size().

Referenced by llvm::PPC::get_VSPLTI_elt(), and llvm::InstCombiner::visitShuffleVectorInst().

static bool isValidElementType ( Type Ty)
static

Predicate for the element types that the SLP vectorizer supports.

The most important thing to filter here are types which are invalid in LLVM vectors. We also filter target specific types which have absolutely no meaningful vectorization path such as x86_fp80 and ppc_f128. This just avoids spending time checking the cost model and realizing that they will be inevitably scalarized.

Definition at line 113 of file SLPVectorizer.cpp.

References llvm::Type::isPPC_FP128Ty(), llvm::VectorType::isValidElementType(), and llvm::Type::isX86_FP80Ty().

Referenced by llvm::slpvectorizer::BoUpSLP::canMapToVector().

static bool matchExtractIndex ( Instruction E,
unsigned  Idx,
unsigned  Opcode 
)
static
Returns
True if Extract{Value,Element} instruction extracts element Idx.

Definition at line 239 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), E, llvm::ExtractValueInst::getNumIndices(), llvm::User::getOperand(), llvm::ConstantInt::getZExtValue(), and llvm::ExtractValueInst::idx_begin().

static bool PhiTypeSorterFunc ( Value V,
Value V2 
)
static

Definition at line 4454 of file SLPVectorizer.cpp.

References llvm::Value::getType().

static void propagateIRFlags ( Value I,
ArrayRef< Value * >  VL 
)
static

Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) that will be converted into a vector (I).

Flag set: NSW, NUW, exact, and all of fast-math.

Definition at line 213 of file SLPVectorizer.cpp.

References i, llvm::NVPTX::PTXLdStInstCode::Scalar, and llvm::ArrayRef< T >::size().

static bool shouldReorderOperands ( int  i,
Instruction I,
SmallVectorImpl< Value * > &  Left,
SmallVectorImpl< Value * > &  Right,
bool  AllSameOpcodeLeft,
bool  AllSameOpcodeRight,
bool  SplatLeft,
bool  SplatRight 
)
static
STATISTIC ( NumVectorInstructions  ,
"Number of vector instructions generated"   
)

Variable Documentation

const unsigned AliasedCheckLimit = 10
static

Definition at line 95 of file SLPVectorizer.cpp.

const char lv_name[] = "SLP Vectorizer"
static

Definition at line 4842 of file SLPVectorizer.cpp.

const unsigned MaxMemDepDistance = 160
static

Definition at line 100 of file SLPVectorizer.cpp.

cl::opt<int> MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static
const int MinScheduleRegionSize = 16
static

If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

Definition at line 104 of file SLPVectorizer.cpp.

cl::opt<unsigned> MinTreeSize("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable"))
static
cl::opt<int> MinVectorRegSizeOption("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static
cl::opt<unsigned> RecursionMaxDepth("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree"))
static
cl::opt<int> ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block"))
static

Limits the size of scheduling regions in a block.

It avoid long compile times for very large blocks where vector instructions are spread over a wide range. This limit is way higher than needed by real-world functions.

cl::opt<bool> ShouldStartVectorizeHorAtStore("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store"))
static
cl::opt<bool> ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions"))
static
cl::opt<int> SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this ""number "))
static