|
LLVM
4.0.0
|
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"#include "llvm/ADT/Optional.h"#include "llvm/ADT/PostOrderIterator.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/Statistic.h"#include "llvm/Analysis/CodeMetrics.h"#include "llvm/Analysis/GlobalsModRef.h"#include "llvm/Analysis/LoopAccessAnalysis.h"#include "llvm/Analysis/ScalarEvolutionExpressions.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/IR/DataLayout.h"#include "llvm/IR/Dominators.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Module.h"#include "llvm/IR/NoFolder.h"#include "llvm/IR/Type.h"#include "llvm/IR/Value.h"#include "llvm/IR/Verifier.h"#include "llvm/Pass.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Transforms/Vectorize.h"#include <algorithm>#include <memory>Go to the source code of this file.
Classes | |
| class | llvm::slpvectorizer::BoUpSLP |
| Bottom Up SLP Vectorizer. More... | |
| struct | llvm::slpvectorizer::BoUpSLP::BlockScheduling::ReadyList |
Namespaces | |
| llvm | |
| Compute iterated dominance frontiers using a linear time algorithm. | |
| llvm::slpvectorizer | |
| A private "module" namespace for types and utilities used by this pass. | |
Macros | |
| #define | SV_NAME "slp-vectorizer" |
| #define | DEBUG_TYPE "SLP" |
Functions | |
| STATISTIC (NumVectorInstructions,"Number of vector instructions generated") | |
| static bool | isValidElementType (Type *Ty) |
| Predicate for the element types that the SLP vectorizer supports. More... | |
| static bool | allSameBlock (ArrayRef< Value * > VL) |
| static bool | allConstant (ArrayRef< Value * > VL) |
| static bool | isSplat (ArrayRef< Value * > VL) |
| static unsigned | getAltOpcode (unsigned Op) |
| static bool | canCombineAsAltInst (unsigned Op) |
| static unsigned | isAltInst (ArrayRef< Value * > VL) |
| static unsigned | getSameOpcode (ArrayRef< Value * > VL) |
| static void | propagateIRFlags (Value *I, ArrayRef< Value * > VL) |
| Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) that will be converted into a vector (I). More... | |
| static bool | allSameType (ArrayRef< Value * > VL) |
| static bool | matchExtractIndex (Instruction *E, unsigned Idx, unsigned Opcode) |
| static bool | InTreeUserNeedToExtract (Value *Scalar, Instruction *UserInst, TargetLibraryInfo *TLI) |
| static MemoryLocation | getLocation (Instruction *I, AliasAnalysis *AA) |
| static bool | isSimple (Instruction *I) |
| static bool | shouldReorderOperands (int i, Instruction &I, SmallVectorImpl< Value * > &Left, SmallVectorImpl< Value * > &Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight, bool SplatLeft, bool SplatRight) |
| static bool | collectValuesToDemote (Value *V, SmallPtrSetImpl< Value * > &Expr, SmallVectorImpl< Value * > &ToDemote, SmallVectorImpl< Value * > &Roots) |
| static bool | hasValueBeenRAUWed (ArrayRef< Value * > VL, ArrayRef< WeakVH > VH, unsigned SliceBegin, unsigned SliceSize) |
| Check that the Values in the slice in VL array are still existent in the WeakVH array. More... | |
| static Value * | createRdxShuffleMask (unsigned VecLen, unsigned NumEltsToRdx, bool IsPairwise, bool IsLeft, IRBuilder<> &Builder) |
| Generate a shuffle mask to be used in a reduction tree. More... | |
| static bool | findBuildVector (InsertElementInst *FirstInsertElem, SmallVectorImpl< Value * > &BuildVector, SmallVectorImpl< Value * > &BuildVectorOpds) |
| Recognize construction of vectors like ra = insertelement <4 x float> undef, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3. More... | |
| static bool | findBuildAggregate (InsertValueInst *IV, SmallVectorImpl< Value * > &BuildVector, SmallVectorImpl< Value * > &BuildVectorOpds) |
| Like findBuildVector, but looks backwards for construction of aggregate. More... | |
| static bool | PhiTypeSorterFunc (Value *V, Value *V2) |
| static Value * | getReductionValue (const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI) |
| Try and get a reduction value from a phi node. More... | |
| static bool | canMatchHorizontalReduction (PHINode *P, BinaryOperator *BI, BoUpSLP &R, TargetTransformInfo *TTI, unsigned MinRegSize) |
| Attempt to reduce a horizontal reduction. More... | |
| Pass * | llvm::createSLPVectorizerPass () |
Variables | |
| static cl::opt< int > | SLPCostThreshold ("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this ""number ")) |
| static cl::opt< bool > | ShouldVectorizeHor ("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions")) |
| static cl::opt< bool > | ShouldStartVectorizeHorAtStore ("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store")) |
| static cl::opt< int > | MaxVectorRegSizeOption ("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) |
| static cl::opt< int > | ScheduleRegionSizeBudget ("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block")) |
| Limits the size of scheduling regions in a block. More... | |
| static cl::opt< int > | MinVectorRegSizeOption ("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) |
| static cl::opt< unsigned > | RecursionMaxDepth ("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree")) |
| static cl::opt< unsigned > | MinTreeSize ("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable")) |
| static const unsigned | AliasedCheckLimit = 10 |
| static const unsigned | MaxMemDepDistance = 160 |
| static const int | MinScheduleRegionSize = 16 |
| If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled. More... | |
| static const char | lv_name [] = "SLP Vectorizer" |
| #define DEBUG_TYPE "SLP" |
Definition at line 51 of file SLPVectorizer.cpp.
| #define SV_NAME "slp-vectorizer" |
Definition at line 50 of file SLPVectorizer.cpp.
VL are constants. Definition at line 137 of file SLPVectorizer.cpp.
References i.
VL are in the same block or false otherwise. Definition at line 120 of file SLPVectorizer.cpp.
References llvm::dyn_cast(), llvm::Instruction::getParent(), I, i, and llvm::ArrayRef< T >::size().
VL have the same type or false otherwise. Definition at line 229 of file SLPVectorizer.cpp.
References llvm::Intrinsic::getType(), i, and llvm::ArrayRef< T >::size().
Referenced by llvm::slpvectorizer::BoUpSLP::buildTree().
Op can be part of an alternate sequence which can later be merged as a ShuffleVector instruction. Definition at line 172 of file SLPVectorizer.cpp.
References llvm::MCID::Add.
Referenced by getSameOpcode().
|
static |
Attempt to reduce a horizontal reduction.
If it is legal to match a horizontal reduction feeding the phi node P with reduction operators BI, then check if it can be done.
Definition at line 4516 of file SLPVectorizer.cpp.
References llvm::PowerOf2Floor(), and ShouldVectorizeHor.
|
static |
Definition at line 3383 of file SLPVectorizer.cpp.
References llvm::MCID::Add, llvm::APIntOps::And(), llvm::SmallPtrSetImpl< PtrType >::count(), llvm::dyn_cast(), llvm::SelectInst::getFalseValue(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), llvm::SelectInst::getTrueValue(), llvm::Value::hasOneUse(), I, llvm::PHINode::incoming_values(), llvm::APIntOps::Or(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::MCID::Select, and llvm::APIntOps::Xor().
Referenced by llvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes().
|
static |
Generate a shuffle mask to be used in a reduction tree.
| VecLen | The length of the vector to be reduced. |
| NumEltsToRdx | The number of elements that should be reduced in the vector. |
| IsPairwise | Whether the reduction is a pairwise or splitting reduction. A pairwise reduction will generate a mask of <0,2,...> or <1,3,..> while a splitting reduction will generate <2,3, undef,undef> for a vector of 4 and NumElts = 2. |
| IsLeft | True will generate a mask of even elements, odd otherwise. |
Definition at line 4071 of file SLPVectorizer.cpp.
References assert(), llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::IRBuilderBase::getInt32(), llvm::IRBuilderBase::getInt32Ty(), and i.
|
static |
Like findBuildVector, but looks backwards for construction of aggregate.
Definition at line 4438 of file SLPVectorizer.cpp.
References llvm::dyn_cast(), llvm::InsertValueInst::getAggregateOperand(), llvm::InsertValueInst::getInsertedValueOperand(), llvm::Value::hasOneUse(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
|
static |
Recognize construction of vectors like ra = insertelement <4 x float> undef, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3.
Returns true if it matches
Definition at line 4406 of file SLPVectorizer.cpp.
References llvm::dyn_cast(), llvm::User::getOperand(), llvm::Value::hasOneUse(), llvm::ARM_PROC::IE, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::Value::use_empty(), and llvm::Instruction::user_back().
Op to create an alternate sequence which can later be merged as a ShuffleVector instruction. Definition at line 154 of file SLPVectorizer.cpp.
References llvm::MCID::Add.
Referenced by isAltInst().
|
static |
Definition at line 279 of file SLPVectorizer.cpp.
References llvm::MemoryLocation::get(), and SI.
|
static |
Try and get a reduction value from a phi node.
Given a phi node P in a block ParentBB, consider possible reductions if they come from either ParentBB or a containing loop latch.
Definition at line 4465 of file SLPVectorizer.cpp.
References llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::LoopInfoBase< N, M >::getLoopFor(), llvm::LoopBase< N, M >::getLoopLatch(), llvm::Instruction::getParent(), and getParent().
VL have the same opcode, or zero. Definition at line 194 of file SLPVectorizer.cpp.
References canCombineAsAltInst(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), I, i, isAltInst(), and llvm::ArrayRef< T >::size().
|
static |
Check that the Values in the slice in VL array are still existent in the WeakVH array.
Vectorization of part of the VL array may cause later values in the VL array to become invalid. We track when this has happened in the WeakVH array.
Definition at line 3729 of file SLPVectorizer.cpp.
References llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::lltok::equal, and llvm::ArrayRef< T >::slice().
|
static |
Definition at line 253 of file SLPVectorizer.cpp.
References llvm::Call, llvm::CallInst::getArgOperand(), llvm::Instruction::getOpcode(), llvm::LoadInst::getPointerOperand(), llvm::StoreInst::getPointerOperand(), llvm::getVectorIntrinsicIDForCall(), llvm::hasVectorInstrinsicScalarOpd(), llvm::SPII::Load, llvm::NVPTX::PTXLdStInstCode::Scalar, SI, and llvm::SPII::Store.
Referenced by llvm::slpvectorizer::BoUpSLP::buildTree().
VL have alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence. (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...) Definition at line 180 of file SLPVectorizer.cpp.
References llvm::dyn_cast(), getAltOpcode(), llvm::Instruction::getOpcode(), I, i, and llvm::ArrayRef< T >::size().
Referenced by getSameOpcode().
|
static |
Definition at line 288 of file SLPVectorizer.cpp.
Referenced by llvm::DOTGraphTraits< DomTreeNode * >::getNodeLabel(), llvm::DOTGraphTraits< RegionNode * >::getNodeLabel(), llvm::DOTGraphTraits< const Function * >::getNodeLabel(), and llvm::DOTGraphTraits< const MachineFunction * >::getNodeLabel().
VL are identical. Definition at line 145 of file SLPVectorizer.cpp.
References i, and llvm::ArrayRef< T >::size().
Referenced by llvm::PPC::get_VSPLTI_elt(), and llvm::InstCombiner::visitShuffleVectorInst().
Predicate for the element types that the SLP vectorizer supports.
The most important thing to filter here are types which are invalid in LLVM vectors. We also filter target specific types which have absolutely no meaningful vectorization path such as x86_fp80 and ppc_f128. This just avoids spending time checking the cost model and realizing that they will be inevitably scalarized.
Definition at line 113 of file SLPVectorizer.cpp.
References llvm::Type::isPPC_FP128Ty(), llvm::VectorType::isValidElementType(), and llvm::Type::isX86_FP80Ty().
Referenced by llvm::slpvectorizer::BoUpSLP::canMapToVector().
|
static |
Definition at line 239 of file SLPVectorizer.cpp.
References assert(), llvm::dyn_cast(), E, llvm::ExtractValueInst::getNumIndices(), llvm::User::getOperand(), llvm::ConstantInt::getZExtValue(), and llvm::ExtractValueInst::idx_begin().
Definition at line 4454 of file SLPVectorizer.cpp.
References llvm::Value::getType().
Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) that will be converted into a vector (I).
Flag set: NSW, NUW, exact, and all of fast-math.
Definition at line 213 of file SLPVectorizer.cpp.
References i, llvm::NVPTX::PTXLdStInstCode::Scalar, and llvm::ArrayRef< T >::size().
|
static |
Definition at line 2033 of file SLPVectorizer.cpp.
References llvm::dyn_cast(), llvm::Instruction::getOpcode(), and llvm::User::getOperand().
| STATISTIC | ( | NumVectorInstructions | , |
| "Number of vector instructions generated" | |||
| ) |
Definition at line 95 of file SLPVectorizer.cpp.
|
static |
Definition at line 4842 of file SLPVectorizer.cpp.
Definition at line 100 of file SLPVectorizer.cpp.
|
static |
Referenced by llvm::slpvectorizer::BoUpSLP::BoUpSLP().
|
static |
If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.
Definition at line 104 of file SLPVectorizer.cpp.
|
static |
Referenced by llvm::slpvectorizer::BoUpSLP::BoUpSLP().
|
static |
|
static |
Limits the size of scheduling regions in a block.
It avoid long compile times for very large blocks where vector instructions are spread over a wide range. This limit is way higher than needed by real-world functions.
|
static |
|
static |
Referenced by canMatchHorizontalReduction().
1.8.6