SLPVectorizer.cpp File Reference

#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "SLPVectorizer/SLPCompatibilityAnalysis.h"
#include "SLPVectorizer/SLPCostAnalysis.h"
#include "SLPVectorizer/SLPMemoryUtils.h"
#include "SLPVectorizer/SLPReductionUtils.h"
#include "SLPVectorizer/SLPShuffleAnalysis.h"
#include "SLPVectorizer/SLPTypeUtils.h"
#include "SLPVectorizer/SLPUtils.h"
#include "VectorizeOptions.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	slpvectorizer::BoUpSLP
	Bottom Up SLP Vectorizer. More...
struct	slpvectorizer::BoUpSLP::StridedPtrInfo
	If we decide to generate strided load / store, this struct contains all the necessary info. More...
struct	slpvectorizer::BoUpSLP::EdgeInfo
	This structure holds any data we need about the edges being traversed during buildTreeRec(). More...
class	slpvectorizer::BoUpSLP::LookAheadHeuristics
	A helper class used for scoring candidates for two consecutive lanes. More...
class	slpvectorizer::BoUpSLP::VLOperands
	A helper data structure to hold the operands of a vector of instructions. More...
struct	llvm::DenseMapInfo< BoUpSLP::EdgeInfo >
struct	llvm::GraphTraits< BoUpSLP * >
struct	llvm::GraphTraits< BoUpSLP * >::ChildIteratorType
	Add the VectorizableTree to the index iterator to be able to return TreeEntry pointers. More...
class	llvm::GraphTraits< BoUpSLP * >::nodes_iterator
	For the node iterator we just need to turn the TreeEntry iterator into a TreeEntry* iterator so that it dereferences to NodeRef. More...
struct	llvm::DOTGraphTraits< BoUpSLP * >
class	slpvectorizer::BoUpSLP::ShuffleCostEstimator
	Merges shuffle masks and emits final shuffle instruction, if required. More...
class	slpvectorizer::BoUpSLP::ShuffleInstructionBuilder
	Merges shuffle masks and emits final shuffle instruction, if required. More...

Macros
#define	SV_NAME   "slp-vectorizer"
#define	DEBUG_TYPE   "SLP"

Functions
	STATISTIC (NumVectorInstructions, "Number of vector instructions generated")
	STATISTIC (NumStridedStoreChains, "Number of vectorized stride stores")
	STATISTIC (NumStoreChains, "Number of vector stores created")
	STATISTIC (NumVectorizedStores, "Number of vectorized stores")
	DEBUG_COUNTER (VectorizedGraphs, "slp-vectorized", "Controls which SLP graphs should be vectorized.")
static bool	areTwoInsertFromSameBuildVector (InsertElementInst *VU, InsertElementInst *V, function_ref< Value *(InsertElementInst *)> GetBaseOperand)
	Check if two insertelement instructions are from the same buildvector.
static LLVM_DUMP_METHOD void	dumpOrder (const BoUpSLP::OrdersType &Order)
static void	gatherPossiblyVectorizableLoads (const BoUpSLP &R, ArrayRef< Value * > VL, const DataLayout &DL, ScalarEvolution &SE, const TargetTransformInfo &TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > &GatheredLoads, bool AddNew=true)
	Tries to find subvector of loads and builds new vector of only loads if can be profitable.
static std::pair< size_t, size_t >	generateKeySubkey (Value *V, const TargetLibraryInfo *TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate)
	Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.
static bool	isMainInstruction (Instruction *I, Instruction *MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.
static SmallVector< Type * >	buildIntrinsicArgTypes (const CallInst *CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo *TTI)
	Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.
static std::pair< InstructionCost, InstructionCost >	getVectorCallCosts (CallInst *CI, Type *VecTy, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, ArrayRef< Type * > ArgTys, const TTI::TargetCostKind CostKind)
	Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.
static InstructionCost	getVectorOpCost (Instruction *I, unsigned VF, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const TTI::TargetCostKind CostKind)
static unsigned	getPoorThroughputTypeKey (Type *Ty)
	Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.
static bool	isPoorThroughputOp (const VectorizeOptions &Opts, Instruction *I, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, PoorThroughputOpCache &Cache, const TTI::TargetCostKind CostKind)
	Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.
static const Loop *	findInnermostNonInvariantLoop (const Loop *L, ArrayRef< Value * > VL)
	Find the innermost loop starting from L, for which at least a single value in VL is not invariant.
static bool	checkEVsForVecCalls (ArrayRef< Value * > VL, const InstructionsState &S, const TargetLibraryInfo &TLI, SmallVectorImpl< unsigned > &Indices, SmallVectorImpl< Value * > &Calls)
	Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.
static bool	allStructUsersAreExtractValueInsts (ArrayRef< Value * > VL)
static std::pair< Instruction *, Instruction * >	getMainAltOpsNoStateVL (ArrayRef< Value * > VL)
	Returns main/alternate instructions for the given VL.
static bool	tryToFindDuplicates (const VectorizeOptions &Opts, SmallVectorImpl< Value * > &VL, SmallVectorImpl< int > &ReuseShuffleIndices, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const InstructionsState &S, const BoUpSLP::EdgeInfo &UserTreeIdx, const BoUpSLP &R, bool BuildGatherOnly=true)
	Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.
static void	scanAssociativeOperands (const InstructionsState &S, DominatorTree &DT, const DataLayout &DL, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const BoUpSLP &R, SmallVectorImpl< BoUpSLP::ValueList > &Operands, SmallBitVector &NegatedColumns, SmallVectorImpl< Value * > &ReassocScalars)
	Peel associative single-use binary chains into operand columns.
static SmallVector< BoUpSLP::ValueList >	alignReassociatedOperandsByKey (ArrayRef< BoUpSLP::ValueList > Operands, function_ref< bool(unsigned Lane, unsigned Col)> IsNegated, const TargetLibraryInfo &TLI)
	Realign flattened operand columns so matching value families share a column across lanes.
static std::tuple< unsigned, unsigned, unsigned, int >	getReassocColumnsQuality (ArrayRef< BoUpSLP::ValueList > Columns, const BoUpSLP &R, unsigned Opcode)
	Rank reassociated operand layouts by vectorizable load columns, other load patterns, broadcast/constant columns, then fewer unique values per column.
static InstructionCost	canConvertToFMA (ArrayRef< Value * > VL, const InstructionsState &S, DominatorTree &DT, const DataLayout &DL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const BoUpSLP &R)
	Check if we can convert fadd/fsub sequence to FMAD.
static bool	isFusableFMulOperand (Value *Op, Instruction *I, const TargetTransformInfo &TTI)
static Instruction *	getFusableFMulOperand (Instruction *I, const TargetTransformInfo &TTI)
static InstructionCost	getUnfusedBinOpCost (Instruction *I, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, TTI::TargetCostKind CostKind)
static unsigned	getLoopTripCount (const VectorizeOptions &Opts, const Loop *L, ScalarEvolution &SE)
	Get the assumed loop trip count for the loop L.
static TTI::VectorInstrContext	getVectorInstrContextHint (ArrayRef< Value * > VL, const APInt &DemandedElts)
	Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.
static bool	isFirstInsertElement (const InsertElementInst *IE1, const InsertElementInst *IE2)
	Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.
template<typename T>
static T *	performExtractsShuffleAction (MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value *Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action)
	Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.
static bool	hasHighRegisterPressure (const VectorizeOptions &Opts, const BasicBlock &BB, Type *Ty, const TargetTransformInfo &TTI)
static Instruction *	propagateMetadata (Instruction *Inst, ArrayRef< Value * > VL)
static DebugLoc	getDebugLocFromPHI (PHINode &PN)
static bool	blockBodyHasVectorInstructions (const VectorizeOptions &Opts, BasicBlock *BB)
	Returns true if BB's body already contains vector instructions, e.g.
static RecurKind	getRdxKind (Value *V)
	Gets recurrence kind from the specified value.
static std::optional< unsigned >	getAggregateSize (Instruction *InsertInst)
static void	findBuildAggregateRec (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, unsigned OperandOffset, const BoUpSLP &R)
static bool	findBuildAggregate (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, const BoUpSLP &R)
	Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.
static Instruction *	getReductionInstr (const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI)
	Try and get a reduction instruction from a phi node.
static Instruction *	tryGetSecondaryReductionRoot (PHINode *Phi, Instruction *Root)
	We could have an initial reduction that is not an add.
static bool	vectorizePackedAggregate (InsertValueInst *IVI, BoUpSLP &R, TargetTransformInfo &TTI, const DataLayout &DL)
	Vectorizes the aggregate IVI, built of the packs of the same layout, as the single vector of all the fields, bitcasted to the vector of the packs.
template<typename T>
static bool	tryToVectorizeSequence (const VectorizeOptions &Opts, SmallVectorImpl< T * > &Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP &R)
template<bool IsCompatibility>
static bool	compareCmp (const VectorizeOptions &Opts, Value *V, Value *V2, TargetLibraryInfo &TLI, const DominatorTree &DT)
	Compare two cmp instructions.
static bool	isNonVectorizableInst (const VectorizeOptions &Opts, const Instruction *I, const TargetLibraryInfo *TLI)
	Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.
template<typename Func>
static void	forEachOperandChainCandidate (const VectorizeOptions &Opts, Instruction *I, Func F, bool ForReduction)
	Visits the value operands of I that are candidates for operand-chain vectorization.

Variables
static const unsigned	AliasedCheckLimit = 10
static const unsigned	MaxMemDepDistance = 160
static const int	MinScheduleRegionSize = 16
	If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.
static const unsigned	MaxPHINumOperands = 128
	Maximum allowed number of operands in the PHI nodes.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "SLP"

Definition at line 124 of file SLPVectorizer.cpp.

SV_NAME

#define SV_NAME   "slp-vectorizer"

Definition at line 123 of file SLPVectorizer.cpp.

Function Documentation

alignReassociatedOperandsByKey()

SmallVector< BoUpSLP::ValueList > alignReassociatedOperandsByKey ( ArrayRef< BoUpSLP::ValueList > Operands, function_ref< bool(unsigned Lane, unsigned Col)> IsNegated, const TargetLibraryInfo & TLI )

static

Realign flattened operand columns so matching value families share a column across lanes.

Lane 0 is canonical; other lanes match by key equality only (never compare hash values). Columns sharing a key are paired by the family of their first operand where available, so e.g. shifts fed by the same load family land in one column instead of pairing by encounter order. Values move between columns only within the same sign: a subtracted leaf never lands in an added column. The sign is queried per lane and column with IsNegated: alternate add/sub nodes negate only the non-leading columns of their subtract lanes.

Definition at line 12061 of file SLPVectorizer.cpp.

References llvm::Aligned, llvm::dyn_cast(), E(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::find(), generateKeySubkey(), llvm::LoadInst::getPointerOperand(), llvm::getUnderlyingObject(), llvm::hash_value(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::map_to_vector(), Operands, P, llvm::seq(), and std::swap().

allStructUsersAreExtractValueInsts()

bool allStructUsersAreExtractValueInsts ( ArrayRef< Value * > VL )

static

Definition at line 9310 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::isa(), and llvm::IsaPred.

areTwoInsertFromSameBuildVector()

bool areTwoInsertFromSameBuildVector ( InsertElementInst * VU, InsertElementInst * V, function_ref< Value *(InsertElementInst *)> GetBaseOperand )

static

Check if two insertelement instructions are from the same buildvector.

Definition at line 6523 of file SLPVectorizer.cpp.

References llvm::dyn_cast_if_present(), llvm::VectorType::getElementCount(), llvm::slpvectorizer::getElementIndex(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::InsertElementInst::getType(), llvm::Value::hasOneUse(), llvm::SmallBitVector::set(), and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost(), and slpvectorizer::BoUpSLP::vectorizeTree().

blockBodyHasVectorInstructions()

bool blockBodyHasVectorInstructions ( const VectorizeOptions & Opts, BasicBlock * BB )

static

Returns true if BB's body already contains vector instructions, e.g.

from an earlier SLP vectorization in the same pass.

Definition at line 25683 of file SLPVectorizer.cpp.

References llvm::any_of(), llvm::slpvectorizer::getValueType(), I, llvm::isa(), llvm::Type::isVectorTy(), and slpvectorizer::BoUpSLP::Opts.

Referenced by slpvectorizer::BoUpSLP::canVersionBlockForRuntimeChecks().

buildIntrinsicArgTypes()

SmallVector< Type * > buildIntrinsicArgTypes ( const CallInst * CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo * TTI )

static

Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.

Definition at line 9030 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), llvm::enumerate(), llvm::IntegerType::get(), llvm::Value::getContext(), llvm::slpvectorizer::getWidenedType(), llvm::isVectorIntrinsicWithScalarOpAtArg(), llvm::Intrinsic::not_intrinsic, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by getVectorOpCost().

canConvertToFMA()

InstructionCost canConvertToFMA ( ArrayRef< Value * > VL, const InstructionsState & S, DominatorTree & DT, const DataLayout & DL, TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const BoUpSLP & R )

static

Check if we can convert fadd/fsub sequence to FMAD.

Returns

Cost of the FMAD, if conversion is possible, invalid cost otherwise.

Definition at line 13833 of file SLPVectorizer.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, llvm::all_of(), llvm::FastMathFlags::allowContract(), Analysis, assert(), llvm::cast(), CostKind, DL, llvm::dyn_cast(), llvm::enumerate(), llvm::FMul, llvm::ArrayRef< T >::front(), llvm::slpvectorizer::InstructionsState::getAltOp(), llvm::slpvectorizer::InstructionsState::getAltOpcode(), getFusableFMulOperand(), llvm::InstructionCost::getInvalid(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::TargetTransformInfo::getOperandInfo(), getUnfusedBinOpCost(), I, llvm::slpvectorizer::InstructionsState::isAddSubLikeOp(), llvm::slpvectorizer::InstructionsState::isAddSubOrFNegLikeOp(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::InstructionsState::isBitwiseLogicOp(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), isFusableFMulOperand(), llvm::slpvectorizer::InstructionsState::isMulDivLikeOp(), llvm::slpvectorizer::InstructionsState::isShiftOp(), llvm::make_filter_range(), llvm::make_isa_range(), Operands, P, llvm::FastMathFlags::set(), std::swap(), llvm::slpvectorizer::InstructionsState::valid(), and llvm::zip().

checkEVsForVecCalls()

bool checkEVsForVecCalls ( ArrayRef< Value * > VL, const InstructionsState & S, const TargetLibraryInfo & TLI, SmallVectorImpl< unsigned > & Indices, SmallVectorImpl< Value * > & Calls )

static

Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.

VL is a bundle whose state S is Instruction::ExtractValue. The bundle is acceptable for widening into one struct-of-vectors call only when:

• every element of VL is an ExtractValueInst,
• every ExtractValueInst extracts the same struct field (its getIndices() matches the main op's indices),
• the aggregate operands form a uniform set of CallInsts (per getSameOpcode) that is not an alt-shuffle and whose return type is a literal struct, and
• every user of every such call is itself an ExtractValueInst, so the external-use extraction code can rebuild scalars via extractvalue + extractelement without needing an insertvalue chain.

On success returns true and fills Indices with the common field index path and Calls with the per-lane aggregate calls (in VL order), for the caller to feed as the operand of the new tree entry. Otherwise returns false and leaves the output parameters untouched.

Definition at line 9266 of file SLPVectorizer.cpp.

References Aggregates, llvm::all_of(), llvm::slpvectorizer::allSameBlock(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::cast(), llvm::ArrayRef< T >::end(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::slpvectorizer::getSameOpcode(), llvm::Value::getType(), llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::IsaPred, IV, and llvm::SmallVectorImpl< T >::swap().

compareCmp()

template<bool IsCompatibility>

bool compareCmp ( const VectorizeOptions & Opts, Value * V, Value * V2, TargetLibraryInfo & TLI, const DominatorTree & DT )

static

Compare two cmp instructions.

If IsCompatibility is true, function returns true if 2 cmps have same/swapped predicates and mos compatible corresponding operands. If IsCompatibility is false, function implements strict weak ordering relation between two cmp instructions, returning true if the first instruction is "less" than the second, i.e. its predicate is less than the predicate of the second or the operands IDs are less than the operands IDs of the second cmp instruction.

Definition at line 35759 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), E(), llvm::DomTreeNodeBase< NodeT >::getDFSNumIn(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), llvm::User::getOperand(), llvm::slpvectorizer::getSameOpcode(), llvm::Type::getScalarSizeInBits(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::Type::getTypeID(), I, llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::isValidElementType(), and slpvectorizer::BoUpSLP::Opts.

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts().

DEBUG_COUNTER()

DEBUG_COUNTER	(	VectorizedGraphs	,
		"slp-vectorized"	,
		"Controls which SLP graphs should be vectorized."	)

dumpOrder()

LLVM_DUMP_METHOD void dumpOrder ( const BoUpSLP::OrdersType & Order )

static

Definition at line 8246 of file SLPVectorizer.cpp.

References llvm::dbgs(), and LLVM_DUMP_METHOD.

findBuildAggregate()

bool findBuildAggregate ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, const BoUpSLP & R )

static

Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.

Also recognize homogeneous aggregates like {<2 x float>, <2 x float>}, {{float, float}, {float, float}}, [2 x {float, float}] and so on. See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples.

Assume LastInsertInst is of InsertElementInst or InsertValueInst type.

Returns

true if it matches.

Definition at line 35059 of file SLPVectorizer.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::erase(), findBuildAggregateRec(), getAggregateSize(), llvm::isa(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by vectorizePackedAggregate().

findBuildAggregateRec()

void findBuildAggregateRec ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, unsigned OperandOffset, const BoUpSLP & R )

static

Definition at line 35020 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), findBuildAggregateRec(), getElementIndex(), llvm::User::getOperand(), llvm::Value::hasOneUse(), and llvm::isa().

Referenced by findBuildAggregate(), and findBuildAggregateRec().

findInnermostNonInvariantLoop()

const Loop * findInnermostNonInvariantLoop ( const Loop * L, ArrayRef< Value * > VL )

static

Find the innermost loop starting from L, for which at least a single value in VL is not invariant.

Definition at line 9216 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), I, and llvm::make_isa_range().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

forEachOperandChainCandidate()

template<typename Func>

void forEachOperandChainCandidate ( const VectorizeOptions & Opts, Instruction * I, Func F, bool ForReduction )

static

Visits the value operands of I that are candidates for operand-chain vectorization.

Definition at line 35952 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::enumerate(), F, I, llvm_unreachable, and slpvectorizer::BoUpSLP::Opts.

Referenced by llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

gatherPossiblyVectorizableLoads()

void gatherPossiblyVectorizableLoads ( const BoUpSLP & R, ArrayRef< Value * > VL, const DataLayout & DL, ScalarEvolution & SE, const TargetTransformInfo & TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > & GatheredLoads, bool AddNew = true )

static

Tries to find subvector of loads and builds new vector of only loads if can be profitable.

Definition at line 8309 of file SLPVectorizer.cpp.

References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::bit_ceil(), llvm::SetVector< T, Vector, Set, N >::contains(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::SmallSet< T, N, C >::contains(), llvm::Data, DL, llvm::SmallVectorImpl< T >::emplace_back(), llvm::ArrayRef< T >::empty(), llvm::SetVector< T, Vector, Set, N >::empty(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::find_if(), llvm::ArrayRef< T >::front(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::LoadInst::getPointerOperand(), llvm::getPointersDiff(), llvm::Value::getType(), llvm::getUnderlyingObject(), llvm::slpvectorizer::getValueType(), llvm::has_single_bit(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SmallSet< T, N, C >::insert(), llvm::SetVector< T, Vector, Set, N >::insert_range(), llvm::slpvectorizer::isValidElementType(), llvm::make_filter_range(), llvm::make_isa_range(), llvm::Offset, P, llvm::seq(), llvm::ArrayRef< T >::size(), and llvm::zip().

generateKeySubkey()

std::pair< size_t, size_t > generateKeySubkey ( Value * V, const TargetLibraryInfo * TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate )

static

Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.

The keys/subkeys can be used for better sorting of the values themselves (keys) and in values subgroups (subkeys).

Definition at line 8934 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), Call, llvm::cast(), llvm::dyn_cast(), generateKeySubkey(), llvm::CmpInst::getInversePredicate(), llvm::VFDatabase::getMappings(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::getVectorIntrinsicIDForCall(), llvm::hash_combine(), llvm::hash_value(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::Instruction::isIntDivRem(), llvm::LoadInst::isSimple(), llvm::isTriviallyVectorizable(), llvm::slpvectorizer::isUndefVector(), llvm::slpvectorizer::isValidForAlternation(), and llvm::slpvectorizer::isVectorLikeInstWithConstOps().

Referenced by alignReassociatedOperandsByKey(), and generateKeySubkey().

getAggregateSize()

std::optional< unsigned > getAggregateSize ( Instruction * InsertInst )

static

Definition at line 34992 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::Type::isSingleValueType(), and IV.

Referenced by findBuildAggregate().

getDebugLocFromPHI()

DebugLoc getDebugLocFromPHI ( PHINode & PN )

static

Definition at line 23751 of file SLPVectorizer.cpp.

References DL, llvm::Instruction::getDebugLoc(), and llvm::DebugLoc::getUnknown().

getFusableFMulOperand()

Instruction * getFusableFMulOperand ( Instruction * I, const TargetTransformInfo & TTI )

static

Returns

the fmul operand of the fadd/fsub I that the backend fuses with I into an fmuladd: the first fusable one from either operand, c - a*b included. I must allow contraction.

Definition at line 13267 of file SLPVectorizer.cpp.

References llvm::cast(), I, isFusableFMulOperand(), llvm::PatternMatch::m_AllowContract(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::PatternMatch::m_FAdd(), llvm::PatternMatch::m_FSub(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by canConvertToFMA().

getLoopTripCount()

unsigned getLoopTripCount ( const VectorizeOptions & Opts, const Loop * L, ScalarEvolution & SE )

static

Get the assumed loop trip count for the loop L.

Definition at line 16163 of file SLPVectorizer.cpp.

References llvm::getLoopEstimatedTripCount(), llvm::ScalarEvolution::getSmallConstantTripCount(), and slpvectorizer::BoUpSLP::Opts.

getMainAltOpsNoStateVL()

std::pair< Instruction *, Instruction * > getMainAltOpsNoStateVL ( ArrayRef< Value * > VL )

static

Returns main/alternate instructions for the given VL.

Unlike getSameOpcode supports non-compatible instructions for better SplitVectorize node support.

Returns

first main/alt instructions, if only poisons and instruction with only 2 opcodes exists. Returns pair of nullptr otherwise.

Definition at line 9908 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), I, llvm::isa(), and llvm::make_filter_range().

getPoorThroughputTypeKey()

unsigned getPoorThroughputTypeKey ( Type * Ty )

static

Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.

fdiv float vs. fdiv double) is cached per width instead of sharing one verdict.

Definition at line 9122 of file SLPVectorizer.cpp.

Referenced by isPoorThroughputOp().

getRdxKind()

RecurKind getRdxKind ( Value * V )

static

Gets recurrence kind from the specified value.

Definition at line 34989 of file SLPVectorizer.cpp.

getReassocColumnsQuality()

std::tuple< unsigned, unsigned, unsigned, int > getReassocColumnsQuality ( ArrayRef< BoUpSLP::ValueList > Columns, const BoUpSLP & R, unsigned Opcode )

static

Rank reassociated operand layouts by vectorizable load columns, other load patterns, broadcast/constant columns, then fewer unique values per column.

Identity columns are scored by the opcode of the emitted combines.

Definition at line 12150 of file SLPVectorizer.cpp.

References llvm::all_of(), slpvectorizer::BoUpSLP::Gather, llvm::slpvectorizer::getReassocCombineOpcode(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::IsaPred, llvm::slpvectorizer::isBinOpIdentityConstant(), llvm::SmallPtrSetImplBase::size(), and slpvectorizer::BoUpSLP::Vectorize.

getReductionInstr()

Instruction * getReductionInstr ( const DominatorTree * DT, PHINode * P, BasicBlock * ParentBB, LoopInfo * LI )

static

Try and get a reduction instruction from a phi node.

Given a phi node P in a block ParentBB, consider possible reductions if they come from either ParentBB or a containing loop latch.

Returns

A candidate reduction value if possible, or

nullptr 

if not possible.

Definition at line 35094 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::isa(), and P.

getUnfusedBinOpCost()

InstructionCost getUnfusedBinOpCost ( Instruction * I, TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, TTI::TargetCostKind CostKind )

static

Returns

the cost of the binary operator I, priced as not fused. No context instruction is passed: targets that model the fmuladd fusion would discount the unfused side of the comparison as well.

Definition at line 13281 of file SLPVectorizer.cpp.

References assert(), CostKind, llvm::TargetTransformInfo::getOperandInfo(), I, llvm::PatternMatch::m_BinOp(), and llvm::PatternMatch::match().

Referenced by canConvertToFMA().

getVectorCallCosts()

std::pair< InstructionCost, InstructionCost > getVectorCallCosts ( CallInst * CI, Type * VecTy, const TargetTransformInfo * TTI, const TargetLibraryInfo * TLI, ArrayRef< Type * > ArgTys, const TTI::TargetCostKind CostKind )

static

Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.

Returns invalid cost for the corresponding calls, if they cannot be vectorized/will be scalarized.

Definition at line 9055 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), CostKind, llvm::dyn_cast(), llvm::VFShape::get(), llvm::ElementCount::getFixed(), llvm::CallBase::getFunctionType(), llvm::InstructionCost::getInvalid(), llvm::slpvectorizer::getNumElements(), llvm::getVectorIntrinsicIDForCall(), llvm::VFDatabase::getVectorizedFunction(), IntrinsicCost, and llvm::CallBase::isNoBuiltin().

Referenced by getVectorOpCost().

getVectorInstrContextHint()

TTI::VectorInstrContext getVectorInstrContextHint ( ArrayRef< Value * > VL, const APInt & DemandedElts )

static

Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.

Definition at line 16479 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::TargetTransformInfo::getVectorInstrContextHint(), I, llvm::isa(), llvm::TargetTransformInfo::None, llvm::seq(), and llvm::ArrayRef< T >::size().

getVectorOpCost()

InstructionCost getVectorOpCost ( Instruction * I, unsigned VF, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const TTI::TargetCostKind CostKind )

static

Returns

the reciprocal-throughput cost of I widened to VF lanes (an arithmetic op or a vectorizable call).

Definition at line 9099 of file SLPVectorizer.cpp.

References assert(), buildIntrinsicArgTypes(), CostKind, llvm::dyn_cast(), llvm::TargetTransformInfo::getOperandInfo(), getVectorCallCosts(), llvm::getVectorIntrinsicIDForCall(), llvm::slpvectorizer::getWidenedType(), I, and llvm::isa().

Referenced by isPoorThroughputOp().

hasHighRegisterPressure()

bool hasHighRegisterPressure ( const VectorizeOptions & Opts, const BasicBlock & BB, Type * Ty, const TargetTransformInfo & TTI )

static

Returns

true if more values with the scalar type Ty than registers are live at some point of the block BB. The blocks larger than the scheduling budget are skipped to save compile time.

Definition at line 20236 of file SLPVectorizer.cpp.

References llvm::SmallPtrSetImpl< PtrType >::erase(), llvm::hasNItemsOrMore(), I, llvm::SmallPtrSetImpl< PtrType >::insert_range(), llvm::isa(), llvm::make_filter_range(), slpvectorizer::BoUpSLP::Opts, llvm::reverse(), and llvm::SmallPtrSetImplBase::size().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

isFirstInsertElement()

bool isFirstInsertElement ( const InsertElementInst * IE1, const InsertElementInst * IE2 )

static

Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.

Definition at line 19016 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), getElementIndex(), llvm::Value::hasOneUse(), and llvm_unreachable.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

isFusableFMulOperand()

bool isFusableFMulOperand ( Value * Op, Instruction * I, const TargetTransformInfo & TTI )

static

Returns

true if Op is a contractable single-use fmul, which the backend fuses with its fadd/fsub user I: Op is in the block of I or the target sinks it there.

Definition at line 13251 of file SLPVectorizer.cpp.

References llvm::any_of(), llvm::FMul, I, llvm::PatternMatch::m_AllowContract(), llvm::PatternMatch::m_FMul(), llvm::PatternMatch::m_Instruction(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by canConvertToFMA(), and getFusableFMulOperand().

isMainInstruction()

bool isMainInstruction ( Instruction * I, Instruction * MainOp, Instruction * AltOp, const TargetLibraryInfo & TLI )

static

Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.

Definition at line 13660 of file SLPVectorizer.cpp.

References llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), and I.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode().

isNonVectorizableInst()

bool isNonVectorizableInst ( const VectorizeOptions & Opts, const Instruction * I, const TargetLibraryInfo * TLI )

static

Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.

Definition at line 35897 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::any_of(), llvm::CallBase::args(), llvm::dyn_cast(), llvm::VFDatabase::getMappings(), getType(), llvm::getVectorIntrinsicIDForCall(), I, II, llvm::isa(), llvm::isTriviallyVectorizable(), llvm::make_isa_range(), and slpvectorizer::BoUpSLP::Opts.

isPoorThroughputOp()

bool isPoorThroughputOp ( const VectorizeOptions & Opts, Instruction * I, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, PoorThroughputOpCache & Cache, const TTI::TargetCostKind CostKind )

static

Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.

fdiv, frem, fsqrt).

Definition at line 9167 of file SLPVectorizer.cpp.

References llvm::any_of(), CostKind, llvm::dyn_cast(), getPoorThroughputTypeKey(), llvm::getVectorIntrinsicIDForCall(), getVectorOpCost(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::slpvectorizer::isValidElementType(), slpvectorizer::BoUpSLP::Opts, and llvm::TargetTransformInfo::TCC_Expensive.

performExtractsShuffleAction()

template<typename T>

T * performExtractsShuffleAction ( MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value * Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action )

static

Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.

It tries to do it in several steps.

1. If the Base vector is not undef vector, resizing the very first mask to have common VF and perform action for 2 input vectors (including non-undef Base). Other shuffle masks are combined with the resulting after the 1 stage and processed as a shuffle of 2 elements.
2. If the Base is undef vector and have only 1 shuffle mask, perform the action only for 1 vector with the given mask, if it is not the identity mask.
3. If > 2 masks are used, perform the remaining shuffle actions for 2 vectors, combing the masks properly between the steps.

Definition at line 19071 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), assert(), llvm::sampleprof::Base, llvm::slpvectorizer::buildUseMask(), E(), I, llvm::slpvectorizer::isUndefVector(), llvm::slpvectorizer::isUndefVector< true >(), llvm::PoisonMaskElem, T, and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

propagateMetadata()

Instruction * propagateMetadata ( Instruction * Inst, ArrayRef< Value * > VL )

static

Returns

I after propagating metadata from VL only for instructions in VL.

Definition at line 23746 of file SLPVectorizer.cpp.

References llvm::make_isa_range(), and llvm::propagateMetadata().

scanAssociativeOperands()

void scanAssociativeOperands ( const InstructionsState & S, DominatorTree & DT, const DataLayout & DL, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const BoUpSLP & R, SmallVectorImpl< BoUpSLP::ValueList > & Operands, SmallBitVector & NegatedColumns, SmallVectorImpl< Value * > & ReassocScalars )

static

Peel associative single-use binary chains into operand columns.

Peeled instructions go to ReassocScalars; sub-operands are inserted after the peeled column so real pairs stay adjacent for flag reuse. Poison lanes are allowed through; a lane that is not itself a matching real instruction can still be peeled as a copyable identity leaf, as long as some other lane anchors the opcode. Add/sub (and fadd/fsub) chains peel together: a peeled subtract keeps the column sign for its first operand and flips it for the second one, recorded per column in NegatedColumns, so subtracted leaves stay out of the positive total.

Definition at line 11967 of file SLPVectorizer.cpp.

References llvm::all_of(), Analysis, llvm::slpvectorizer::InstructionsState::areInstructionsWithCopyableElements(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), DL, llvm::dyn_cast(), llvm::SmallBitVector::empty(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::slpvectorizer::getReassocCombineOpcode(), I, llvm::SmallVectorImpl< T >::insert(), llvm::slpvectorizer::InstructionsState::invalid(), llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::slpvectorizer::isReassocChainLink(), Operands, llvm::SmallBitVector::push_back(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

STATISTIC() [1/4]

STATISTIC	(	NumStoreChains	,
		"Number of vector stores created"	)

STATISTIC() [2/4]

STATISTIC	(	NumStridedStoreChains	,
		"Number of vectorized stride stores"	)

STATISTIC() [3/4]

STATISTIC	(	NumVectorInstructions	,
		"Number of vector instructions generated"	)

STATISTIC() [4/4]

STATISTIC	(	NumVectorizedStores	,
		"Number of vectorized stores"	)

tryGetSecondaryReductionRoot()

Instruction * tryGetSecondaryReductionRoot ( PHINode * Phi, Instruction * Root )

static

We could have an initial reduction that is not an add.

r *= v1 + v2 + v3 + v4 In such a case start looking for a tree rooted in the first '+'. \Returns the new root if found, which may be nullptr if not an instruction.

Definition at line 35142 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::User::getOperand(), llvm::isa(), LHS, and RHS.

tryToFindDuplicates()

bool tryToFindDuplicates ( const VectorizeOptions & Opts, SmallVectorImpl< Value * > & VL, SmallVectorImpl< int > & ReuseShuffleIndices, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const InstructionsState & S, const BoUpSLP::EdgeInfo & UserTreeIdx, const BoUpSLP & R, bool BuildGatherOnly = true )

static

Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.

The list of unique scalars is extended by poison values to the whole register size.

Returns

false if VL could not be uniquified, in which case VL is unchanged and ReuseShuffleIndices is empty.

Definition at line 9949 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::any_of(), llvm::slpvectorizer::InstructionsState::areInstructionsWithCopyableElements(), assert(), slpvectorizer::BoUpSLP::BlendedLoadVectorize, llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::APInt::clearBit(), slpvectorizer::BoUpSLP::CompressVectorize, llvm::slpvectorizer::computeCommonAlignment(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), CostKind, llvm::dbgs(), llvm::slpvectorizer::doesNotNeedToSchedule(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), llvm::SmallVectorImpl< T >::erase(), llvm::find_if_not(), llvm::SmallVectorTemplateCommon< T, typename >::front(), slpvectorizer::BoUpSLP::Gather, llvm::APInt::getAllOnes(), llvm::Instruction::getDataLayout(), llvm::slpvectorizer::getFullVectorNumberOfElements(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::User::getNumOperands(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::TargetTransformInfo::getOperandInfo(), llvm::slpvectorizer::getScalarizationOverhead(), llvm::slpvectorizer::getShuffleCost(), llvm::DataLayout::getTypeSizeInBits(), llvm::slpvectorizer::getValueType(), llvm::slpvectorizer::getWidenedType(), llvm::APInt::getZero(), llvm::has_single_bit(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), InstructionCost, llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::IsaPred, isConstant(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::slpvectorizer::isVectorLikeInstWithConstOps(), LLVM_DEBUG, llvm::make_filter_range(), llvm::make_range(), N, slpvectorizer::BoUpSLP::Opts, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::rbegin(), llvm::SmallVectorTemplateCommon< T, typename >::rend(), slpvectorizer::BoUpSLP::ScatterVectorize, llvm::seq(), llvm::APInt::setBit(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, slpvectorizer::BoUpSLP::StridedVectorize, llvm::TargetTransformInfo::TCC_Expensive, llvm::TargetTransformInfo::TCC_Free, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace(), slpvectorizer::BoUpSLP::EdgeInfo::UserTE, and slpvectorizer::BoUpSLP::Vectorize.

tryToVectorizeSequence()

template<typename T>

bool tryToVectorizeSequence ( const VectorizeOptions & Opts, SmallVectorImpl< T * > & Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP & R )

static

\Returns the minimum number of elements that we will attempt to vectorize.

Definition at line 35638 of file SLPVectorizer.cpp.

References llvm::ArrayRef(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), Changed, llvm::SmallVectorImpl< T >::clear(), llvm::dbgs(), llvm::dyn_cast(), E(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::End, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::slpvectorizer::getValueType(), I, llvm::isa(), llvm::slpvectorizer::isValidElementType(), LLVM_DEBUG, slpvectorizer::BoUpSLP::Opts, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::stable_sort(), llvm::SmallVectorImpl< T >::swap(), and T.

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts(), and llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

vectorizePackedAggregate()

bool vectorizePackedAggregate ( InsertValueInst * IVI, BoUpSLP & R, TargetTransformInfo & TTI, const DataLayout & DL )

static

Vectorizes the aggregate IVI, built of the packs of the same layout, as the single vector of all the fields, bitcasted to the vector of the packs.

The packs vectorized alone would leave the vectors narrower than possible.

Definition at line 35457 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::ArrayRef(), assert(), llvm::cast(), CostKind, llvm::dbgs(), DL, llvm::dyn_cast(), llvm::enumerate(), F, findBuildAggregate(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::APInt::getAllOnes(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::slpvectorizer::getShuffleCost(), llvm::Value::getType(), llvm::slpvectorizer::getWidenedType(), I, llvm::detail::DenseSetImpl< ValueT, MapTy >::insert_range(), llvm::is_contained(), llvm::isa(), llvm::IsaPred, llvm::ShuffleVectorInst::isIdentityMask(), LLVM_DEBUG, llvm::slpvectorizer::matchPackedFields(), llvm::Instruction::moveBefore(), llvm::Next, llvm::TargetTransformInfo::None, llvm::SmallBitVector::set(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, and llvm::SmallBitVector::test().

Variable Documentation

AliasedCheckLimit

const unsigned AliasedCheckLimit = 10

static

Definition at line 136 of file SLPVectorizer.cpp.

MaxMemDepDistance

const unsigned MaxMemDepDistance = 160

static

Definition at line 141 of file SLPVectorizer.cpp.

Referenced by slpvectorizer::BoUpSLP::canVersionForRuntimeChecks().

MaxPHINumOperands

const unsigned MaxPHINumOperands = 128

static

Maximum allowed number of operands in the PHI nodes.

Definition at line 148 of file SLPVectorizer.cpp.

MinScheduleRegionSize

const int MinScheduleRegionSize = 16

static

If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

Definition at line 145 of file SLPVectorizer.cpp.

Referenced by slpvectorizer::BoUpSLP::::HorizontalReduction.