A private "module" namespace for types and utilities used by this pass. More...

Classes
class	BinOpSameOpcodeHelper
	Helper class that determines VL can use the same opcode. More...
class	BoUpSLP
	Bottom Up SLP Vectorizer. More...

Enumerations
enum class	UseMask { FirstArg , SecondArg , UndefsAsMask }
	Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask. More...

Functions
bool	isValidForAlternation (unsigned Opcode)
InstructionCost	getShuffleCost (const TargetTransformInfo &TTI, TargetTransformInfo::ShuffleKind Kind, VectorType Tp, ArrayRef< int > Mask={}, TargetTransformInfo::TargetCostKind CostKind=TargetTransformInfo::TCK_RecipThroughput, int Index=0, VectorType SubTp=nullptr, ArrayRef< const Value * > Args={})
	Returns the cost of the shuffle instructions with the given `Kind`, vector type `Tp` and optional `Mask`.
std::pair< InstructionCost, InstructionCost >	getGEPCosts (const TargetTransformInfo &TTI, ArrayRef< Value * > Ptrs, Value BasePtr, unsigned Opcode, TargetTransformInfo::TargetCostKind CostKind, Type ScalarTy, VectorType *VecTy)
	Calculate the scalar and the vector costs from vectorizing set of GEPs.
InstructionCost	getBlendedLoadCost (const TargetTransformInfo &TTI, Type *VecTy, Align Alignment, unsigned AddressSpace, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of a BlendedLoadVectorize node loading `VecTy:` two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.
bool	isConstant (Value *V)
bool	isVectorLikeInstWithConstOps (Value *V)
	Checks if `V` is one of vector-like instructions, i.e.
unsigned	getNumElements (Type *Ty)
unsigned	getPartNumElems (unsigned Size, unsigned NumParts)
	Returns power-of-2 number of elements in a single register (part), given the total number of elements `Size` and number of registers (parts) `NumParts`.
unsigned	getNumElems (unsigned Size, unsigned PartNumElems, unsigned Part)
	Returns correct remaining number of elements, considering total amount `Size`, (power-of-2 number) of elements in a single register `PartNumElems` and current register (part) `Part`.
std::string	shortBundleName (ArrayRef< Value * > VL, int Idx=-1)
	Print a short descriptor of the instruction bundle suitable for debug output.
bool	allSameBlock (ArrayRef< Value * > VL)
bool	allConstant (ArrayRef< Value * > VL)
bool	isSplat (ArrayRef< Value * > VL)
Intrinsic::ID	isEquivalentIntrinsicID (Intrinsic::ID LHS, Intrinsic::ID RHS)
	Checks if `LHS` and `RHS` are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.
bool	isCommutative (const Instruction I, const Value ValWithUses, bool IsCopyable)
bool	isCommutative (const Instruction *I)
	This is a helper function to check whether `I` is commutative.
bool	isCommutableOperand (const Instruction I, Value ValWithUses, unsigned Op, bool IsCopyable=false)
	Checks if the operand is commutative.
unsigned	getNumberOfPotentiallyCommutativeOps (Instruction *I)
std::optional< unsigned >	getElementIndex (const Value *Inst, unsigned Offset)
bool	allSameOpcode (ArrayRef< Value * > VL)
std::optional< unsigned >	getExtractIndex (const Instruction *E)
void	inversePermutation (ArrayRef< unsigned > Indices, SmallVectorImpl< int > &Mask)
	Compute the inverse permutation `Mask` of `Indices`.
void	reorderScalars (SmallVectorImpl< Value * > &Scalars, ArrayRef< int > Mask)
	Reorders the list of scalars in accordance with the given `Mask`.
bool	allSameType (ArrayRef< Value * > VL)
template<typename T>
std::optional< unsigned >	getInsertExtractIndex (const Value *Inst, unsigned Offset)
template std::optional< unsigned >	getInsertExtractIndex< InsertElementInst > (const Value *, unsigned)
template std::optional< unsigned >	getInsertExtractIndex< ExtractElementInst > (const Value *, unsigned)
bool	areAllOperandsNonInsts (Value *V)
	Checks if the provided value does not require scheduling.
bool	isUsedOutsideBlock (Value *V)
	Checks if the provided value does not require scheduling.
bool	doesNotNeedToBeScheduled (Value *V)
	Checks if the specified value does not require scheduling.
bool	doesNotNeedToSchedule (ArrayRef< Value * > VL)
	Checks if the specified array of instructions does not require scheduling.
void	transformScalarShuffleIndiciesToVector (unsigned VecTyNumElements, SmallVectorImpl< int > &Mask)
unsigned	getShufflevectorNumGroups (ArrayRef< Value * > VL)
SmallVector< int >	calculateShufflevectorMask (ArrayRef< Value * > VL)
SmallBitVector	buildUseMask (int VF, ArrayRef< int > Mask, UseMask MaskArg)
	Prepares a use bitset for the given mask either for the first argument or for the second.
template<bool IsPoisonOnly>
SmallBitVector	isUndefVector (const Value *V, const SmallBitVector &UseMask={})
	Checks if the given value is actually an undefined constant vector.
template SmallBitVector	isUndefVector< false > (const Value *, const SmallBitVector &)
template SmallBitVector	isUndefVector< true > (const Value *, const SmallBitVector &)
bool	doesInTreeUserNeedToExtract (Value Scalar, Instruction UserInst, TargetLibraryInfo TLI, const TargetTransformInfo TTI)
MemoryLocation	getLocation (Instruction *I)
bool	isSimple (Instruction *I)
bool	isSelectedBaseLoad (Type ScalarTy, ArrayRef< Value > PointerOps, const DataLayout &DL, Value &TrueBase, Value &FalseBase, SmallVectorImpl< Value * > &Conditions)
	Checks if the loads with scalar type `ScalarTy` and pointer operands `PointerOps` are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?
void	addMask (SmallVectorImpl< int > &Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs=false)
	Shuffles `Mask` in accordance with the given `SubMask`.
void	fixupOrderingIndices (MutableArrayRef< unsigned > Order)
	Order may have elements assigned special value (size) which is out of bounds.
SmallBitVector	getAltInstrMask (ArrayRef< Value * > VL, Type *ScalarTy, unsigned Opcode0, unsigned Opcode1)
SmallVector< Constant * >	replicateMask (ArrayRef< Constant * > Val, unsigned VF)
	Replicates the given `Val` `VF` times.
Intrinsic::ID	getMaskedDivRemIntrinsic (unsigned Opcode)

Variables
constexpr int	UsesLimit = 64
	Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Detailed Description

A private "module" namespace for types and utilities used by this pass.

These are implementation details and should not be used by clients.

Enumeration Type Documentation

◆ UseMask

enum class llvm::slpvectorizer::UseMask

strong

Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask.

Enumerator

FirstArg

The mask is expected to be for permutation of 1-2 vectors, check for the mask elements for the first argument (mask indices are in range [0:VF)).

SecondArg

The mask is expected to be for permutation of 2 vectors, check for the mask elements for the second argument (mask indices are in range [VF:2*VF))

UndefsAsMask

Consider undef mask elements (-1) as placeholders for future shuffle elements and mark them as ones as being used in future.

Non-undef elements are considered as unused since they're already marked as used in the mask.

Definition at line 224 of file SLPUtils.h.

Function Documentation

◆ addMask()

void llvm::slpvectorizer::addMask	(	SmallVectorImpl< int > &	Mask,
		ArrayRef< int >	SubMask,
		bool	ExtendingManyInputs = false )

Shuffles Mask in accordance with the given SubMask.

Parameters

ExtendingManyInputs Supports reshuffling of the mask with not only one but two input vectors.

Definition at line 592 of file SLPUtils.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ allConstant()

bool llvm::slpvectorizer::allConstant ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are constants (but not globals/constant expressions).

Definition at line 107 of file SLPUtils.cpp.

References llvm::all_of(), and isConstant().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getSpillCost(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameBlock()

bool llvm::slpvectorizer::allSameBlock ( ArrayRef< Value * > VL )

Returns: True if all of the instructions in VL are in the same block.

Definition at line 85 of file SLPUtils.cpp.

References llvm::all_of(), llvm::cast(), llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), II, llvm::isa(), llvm::IsaPred, isVectorLikeInstWithConstOps(), and llvm::iterator_range().

Referenced by checkEVsForVecCalls(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameOpcode()

bool llvm::slpvectorizer::allSameOpcode ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL use the same opcode. For comparison instructions, also checks if predicates match. PoisonValues are considered matching. Interchangeable instructions are not considered.

Definition at line 230 of file SLPUtils.cpp.

References llvm::all_of(), llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::Instruction::getOpcode(), llvm::isa(), llvm::IsaPred, and llvm::make_range().

Referenced by getSameOpcode().

◆ allSameType()

bool llvm::slpvectorizer::allSameType ( ArrayRef< Value * > VL )

Returns: True iff every value in VL has the same Type as the first.

Definition at line 291 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::consume_front(), and llvm::ArrayRef< T >::empty().

Referenced by slpvectorizer::BoUpSLP::buildTree(), slpvectorizer::BoUpSLP::buildTree(), and slpvectorizer::BoUpSLP::getReorderingData().

◆ areAllOperandsNonInsts()

bool llvm::slpvectorizer::areAllOperandsNonInsts ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all operands are either not instructions or phi nodes or instructions from different blocks.

Definition at line 331 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and llvm::mayHaveNonDefUseDependency().

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ buildUseMask()

SmallBitVector llvm::slpvectorizer::buildUseMask	(	int	VF,
		ArrayRef< int >	Mask,
		UseMask	MaskArg )

Prepares a use bitset for the given mask either for the first argument or for the second.

Definition at line 439 of file SLPUtils.cpp.

References llvm::enumerate(), FirstArg, llvm::PoisonMaskElem, SecondArg, and UndefsAsMask.

Referenced by performExtractsShuffleAction().

◆ calculateShufflevectorMask()

SmallVector< int > llvm::slpvectorizer::calculateShufflevectorMask ( ArrayRef< Value * > VL )

Returns: a shufflevector mask which is used to vectorize shufflevectors e.g., %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> the result is <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>

Definition at line 422 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::ArrayRef< T >::front(), getShufflevectorNumGroups(), and llvm::PoisonMaskElem.

◆ doesInTreeUserNeedToExtract()

bool llvm::slpvectorizer::doesInTreeUserNeedToExtract	(	Value *	Scalar,
		Instruction *	UserInst,
		TargetLibraryInfo *	TLI,
		const TargetTransformInfo *	TTI )

Returns: True if in-tree use also needs extract. This refers to possible scalar operand in vectorized instruction.

Definition at line 506 of file SLPUtils.cpp.

References llvm::any_of(), llvm::CallBase::args(), llvm::cast(), llvm::enumerate(), llvm::Instruction::getOpcode(), llvm::LoadInst::getPointerOperand(), and llvm::getVectorIntrinsicIDForCall().

Referenced by slpvectorizer::BoUpSLP::buildExternalUses().

◆ doesNotNeedToBeScheduled()

bool llvm::slpvectorizer::doesNotNeedToBeScheduled ( Value * V )

Checks if the specified value does not require scheduling.

It does not require scheduling if all operands and all users do not need to be scheduled in the current basic block.

Definition at line 358 of file SLPUtils.cpp.

References areAllOperandsNonInsts(), and isUsedOutsideBlock().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ doesNotNeedToSchedule()

bool llvm::slpvectorizer::doesNotNeedToSchedule ( ArrayRef< Value * > VL )

Checks if the specified array of instructions does not require scheduling.

It is so if all either instructions have operands that do not require scheduling or their users do not require scheduling since they are phis or in other basic blocks.

Definition at line 362 of file SLPUtils.cpp.

References llvm::all_of(), areAllOperandsNonInsts(), llvm::ArrayRef< T >::empty(), and isUsedOutsideBlock().

Referenced by tryToFindDuplicates().

◆ fixupOrderingIndices()

void llvm::slpvectorizer::fixupOrderingIndices ( MutableArrayRef< unsigned > Order )

Order may have elements assigned special value (size) which is out of bounds.

Such indices only appear on places which correspond to undef values (see canReuseExtract for details) and used in order to avoid undef values have effect on operands ordering. The first loop below simply finds all unused indices and then the next loop nest assigns these indices for undef values positions. As an example below Order has two undef positions and they have assigned values 3 and 7 respectively: before: 6 9 5 4 9 2 1 0 after: 6 3 5 4 7 2 1 0

Definition at line 617 of file SLPUtils.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::SmallBitVector::find_first(), llvm::SmallBitVector::find_next(), I, llvm::SmallBitVector::none(), llvm::SmallBitVector::reset(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getAltInstrMask()

SmallBitVector llvm::slpvectorizer::getAltInstrMask	(	ArrayRef< Value * >	VL,
		Type *	ScalarTy,
		unsigned	Opcode0,
		unsigned	Opcode1 )

Returns: a bitset for selecting opcodes. false for Opcode0 and true for Opcode1.

Definition at line 641 of file SLPUtils.cpp.

References llvm::cast(), getNumElements(), llvm::isa(), llvm::seq(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getBlendedLoadCost()

InstructionCost llvm::slpvectorizer::getBlendedLoadCost	(	const TargetTransformInfo &	TTI,
		Type *	VecTy,
		Align	Alignment,
		unsigned	AddressSpace,
		TargetTransformInfo::TargetCostKind	CostKind )

Returns the cost of a BlendedLoadVectorize node loading VecTy: two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.

The blend mask is a separate operand node, so its cost is counted there, not here.

Definition at line 131 of file SLPCostAnalysis.cpp.

References llvm::CmpInst::BAD_ICMP_PREDICATE, CostKind, and llvm::CmpInst::makeCmpResultType().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ getElementIndex()

std::optional< unsigned > llvm::slpvectorizer::getElementIndex	(	const Value *	Inst,
		unsigned	Offset = 0 )

Returns: inserting or extracting index of InsertElement, ExtractElement or InsertValue instruction, using Offset as base offset for index.; std::nullopt if the index is not an immediate.

Definition at line 202 of file SLPUtils.cpp.

References llvm::dyn_cast(), getInsertExtractIndex(), I, IV, and llvm::Offset.

Referenced by areTwoInsertFromSameBuildVector(), slpvectorizer::BoUpSLP::getReorderingData(), and isUndefVector().

◆ getExtractIndex()

std::optional< unsigned > llvm::slpvectorizer::getExtractIndex ( const Instruction * E )

Returns: Optional element Idx for Extract{Value,Element} instructions.

Definition at line 248 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), and getNumElements().

Referenced by slpvectorizer::BoUpSLP::getReorderingData().

◆ getGEPCosts()

std::pair< InstructionCost, InstructionCost > llvm::slpvectorizer::getGEPCosts	(	const TargetTransformInfo &	TTI,
		ArrayRef< Value * >	Ptrs,
		Value *	BasePtr,
		unsigned	Opcode,
		TTI::TargetCostKind	CostKind,
		Type *	ScalarTy,
		VectorType *	VecTy )

Calculate the scalar and the vector costs from vectorizing set of GEPs.

Definition at line 51 of file SLPCostAnalysis.cpp.

References llvm::all_of(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::IsaPred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::TargetTransformInfo::TCC_Free.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

◆ getInsertExtractIndex()

template<typename T>

std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex	(	const Value *	Inst,
		unsigned	Offset )

Returns: inserting or extracting index of InsertElement / ExtractElement instruction, using Offset as base offset for index. Only instantiated for InsertElementInst and ExtractElementInst (see SLPUtils.cpp).

Definition at line 298 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::Offset.

Referenced by getElementIndex().

◆ getInsertExtractIndex< ExtractElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< ExtractElementInst >	(	const Value *	,
		unsigned	)

◆ getInsertExtractIndex< InsertElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< InsertElementInst >	(	const Value *	,
		unsigned	)

◆ getLocation()

MemoryLocation llvm::slpvectorizer::getLocation ( Instruction * I )

Returns: the AA location that is being access by the instruction.

Definition at line 534 of file SLPUtils.cpp.

References llvm::dyn_cast(), llvm::MemoryLocation::get(), and I.

◆ getMaskedDivRemIntrinsic()

Intrinsic::ID llvm::slpvectorizer::getMaskedDivRemIntrinsic ( unsigned Opcode )

Returns: the masked division/remainder intrinsic corresponding to Opcode. Disabled lanes of these intrinsics are poison rather than UB, unlike the plain opcode.

Definition at line 664 of file SLPUtils.cpp.

References llvm_unreachable.

Referenced by getMaskedDivRemCost().

◆ getNumberOfPotentiallyCommutativeOps()

unsigned llvm::slpvectorizer::getNumberOfPotentiallyCommutativeOps ( Instruction * I )

Returns: number of operands of I, considering commutativity. Returns 2 for commutative intrinsics.

Parameters

I	The instruction to check for commutativity

Definition at line 192 of file SLPUtils.cpp.

References I, llvm::isa(), and isCommutative().

◆ getNumElements()

unsigned llvm::slpvectorizer::getNumElements ( Type * Ty )

Returns: the number of elements for Ty.

Definition at line 58 of file SLPUtils.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::getVectorizedTypeVF(), llvm::isa(), and llvm::isVectorizedTy().

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate(), getAltInstrMask(), getExtractIndex(), getExtractWithExtendCost(), getNumberOfParts(), getVectorCallCosts(), getWidenedType(), and slpvectorizer::BoUpSLP::vectorizeTree().

◆ getNumElems()

unsigned llvm::slpvectorizer::getNumElems	(	unsigned	Size,
		unsigned	PartNumElems,
		unsigned	Part )

Returns correct remaining number of elements, considering total amount Size, (power-of-2 number) of elements in a single register PartNumElems and current register (part) Part.

Definition at line 70 of file SLPUtils.cpp.

References Size.

Referenced by slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getPartNumElems()

unsigned llvm::slpvectorizer::getPartNumElems	(	unsigned	Size,
		unsigned	NumParts )

Returns power-of-2 number of elements in a single register (part), given the total number of elements Size and number of registers (parts) NumParts.

Definition at line 66 of file SLPUtils.cpp.

References llvm::bit_ceil(), llvm::divideCeil(), and Size.

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getShuffleCost()

InstructionCost llvm::slpvectorizer::getShuffleCost	(	const TargetTransformInfo &	TTI,
		TargetTransformInfo::ShuffleKind	Kind,
		VectorType *	Tp,
		ArrayRef< int >	Mask = {},
		TargetTransformInfo::TargetCostKind	CostKind = TargetTransformInfo::TCK_RecipThroughput,
		int	Index = 0,
		VectorType *	SubTp = nullptr,
		ArrayRef< const Value * >	Args = {} )

Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.

Adds SLP-specific cost estimation for insert subvector pattern.

Definition at line 25 of file SLPCostAnalysis.cpp.

References CostKind, llvm::FixedVectorType::get(), llvm::VectorType::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::Type::getScalarType(), llvm::ShuffleVectorInst::isInsertSubvectorMask(), llvm::TargetTransformInfo::SK_InsertSubvector, llvm::TargetTransformInfo::SK_PermuteTwoSrc, and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canVectorizeLoads(), getExtractWithExtendCost(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getTreeCost(), getVectorInstrCost(), isMaskedLoadCompress(), and tryToFindDuplicates().

◆ getShufflevectorNumGroups()

unsigned llvm::slpvectorizer::getShufflevectorNumGroups ( ArrayRef< Value * > VL )

Returns

the number of groups of shufflevector A group has the following features

All of value in a group are shufflevector.
The mask of all shufflevector is isExtractSubvectorMask.
The mask of all shufflevector uses all of the elements of the source. e.g., it is 1 group (%0) %1 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %2 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> it is 2 groups (%3 and %4) %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> it is 0 group %12 = shufflevector <8 x i16> %10, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %13 = shufflevector <8 x i16> %11, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>

Definition at line 382 of file SLPUtils.cpp.

References llvm::SmallBitVector::all(), llvm::all_of(), assert(), llvm::cast(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), I, llvm::IsaPred, llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().

Referenced by calculateShufflevectorMask().

◆ inversePermutation()

void llvm::slpvectorizer::inversePermutation	(	ArrayRef< unsigned >	Indices,
		SmallVectorImpl< int > &	Mask )

Compute the inverse permutation Mask of Indices.

Definition at line 272 of file SLPUtils.cpp.

References I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::addOrdered(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getReorderingData(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ isCommutableOperand()

bool llvm::slpvectorizer::isCommutableOperand	(	const Instruction *	I,
		Value *	ValWithUses,
		unsigned	Op,
		bool	IsCopyable = false )

Checks if the operand is commutative.

In commutative operations, not all operands might commutable, e.g. for fmuladd only 2 first operands are commutable.

Definition at line 174 of file SLPUtils.cpp.

References assert(), llvm::dyn_cast(), I, llvm::isa(), and isCommutative().

◆ isCommutative() [1/2]

bool llvm::slpvectorizer::isCommutative ( const Instruction * I )

This is a helper function to check whether I is commutative.

This is a convenience wrapper that calls the two-parameter version of isCommutative with the same instruction for both parameters. This is the common case where the instruction being checked for commutativity is the same as the instruction whose uses are analyzed for special patterns (see the two-parameter version above for details).

Parameters

I	The instruction to check for commutativity

Returns: true if the instruction is commutative, false otherwise

Definition at line 172 of file SLPUtils.cpp.

References I, and isCommutative().

◆ isCommutative() [2/2]

bool llvm::slpvectorizer::isCommutative	(	const Instruction *	I,
		const Value *	ValWithUses,
		bool	IsCopyable = false )

Returns: True if I is commutative, handles CmpInst and BinaryOperator. For BinaryOperator, it also checks if ValWithUses is used in specific patterns that make it effectively commutative (like equality comparisons with zero). In most cases, users should not call this function directly (since I and ValWithUses are the same). However, when analyzing interchangeable instructions, we need to use the converted opcode along with the original uses.

Parameters

I	The instruction to check for commutativity
ValWithUses	The value whose uses are analyzed for special patterns

Definition at line 137 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::Value::hasNUsesOrMore(), llvm::Value::hasUseList(), I, llvm::Value::uses(), and UsesLimit.

Referenced by getNumberOfPotentiallyCommutativeOps(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getScoreAtLevelRec(), isCommutableOperand(), isCommutative(), and slpvectorizer::BoUpSLP::isProfitableToReorder().

◆ isConstant()

bool llvm::slpvectorizer::isConstant ( Value * V )

Returns: True if the value is a constant (but not globals/constant expressions).

Definition at line 34 of file SLPUtils.cpp.

References llvm::isa().

Referenced by allConstant(), areCompatibleCmpOps(), arePointersCompatible(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and isVectorLikeInstWithConstOps().

◆ isEquivalentIntrinsicID()

Intrinsic::ID llvm::slpvectorizer::isEquivalentIntrinsicID	(	Intrinsic::ID	LHS,
		Intrinsic::ID	RHS )

Checks if LHS and RHS are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.

Returns: the intrinsic ID to use for the pair (RHS if the IDs match, otherwise Intrinsic::fma), or Intrinsic::not_intrinsic if they are not equivalent.

Definition at line 128 of file SLPUtils.cpp.

References llvm::Intrinsic::not_intrinsic.

Referenced by getSameOpcode().

◆ isSelectedBaseLoad()

bool llvm::slpvectorizer::isSelectedBaseLoad	(	Type *	ScalarTy,
		ArrayRef< Value * >	PointerOps,
		const DataLayout &	DL,
		Value *&	TrueBase,
		Value *&	FalseBase,
		SmallVectorImpl< Value * > &	Conditions )

Checks if the loads with scalar type ScalarTy and pointer operands PointerOps are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?

A[i] : B[i] takes. On success TrueBase / FalseBase are the candidate bases and Conditions holds each lane's select condition, used to build the blend mask. Lane Idx must be at Base + Idx * sizeof(ScalarTy); only dense, natural lane order starting at the base is recognized (reordered or partial groups fall back to Gather/Scatter).

Definition at line 552 of file SLPUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::sampleprof::Base, DL, llvm::dyn_cast(), llvm::enumerate(), F, GEP, llvm::APInt::getZExtValue(), llvm::APInt::isNegative(), llvm::Offset, P, llvm::ArrayRef< T >::size(), and T.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ isSimple()

bool llvm::slpvectorizer::isSimple ( Instruction * I )

Returns: True if the instruction is not a volatile or atomic load/store.

Definition at line 542 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, and MI.

◆ isSplat()

bool llvm::slpvectorizer::isSplat ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are identical or some of them are UndefValue.

Definition at line 113 of file SLPUtils.cpp.

References llvm::isa().

Referenced by llvm::DOTGraphTraits< BoUpSLP * >::getNodeLabel(), and slpvectorizer::BoUpSLP::getReorderingData().

◆ isUndefVector()

template<bool IsPoisonOnly>

SmallBitVector llvm::slpvectorizer::isUndefVector	(	const Value *	V,
		const SmallBitVector &	UseMask = {} )

Checks if the given value is actually an undefined constant vector.

Also, if the UseMask is not empty, tries to check if the non-masked elements actually mask the insertelement buildvector, if any.

Definition at line 456 of file SLPUtils.cpp.

References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::dyn_cast(), getElementIndex(), I, II, llvm::isa(), isUndefVector(), llvm::SmallBitVector::reset(), and T.

Referenced by generateKeySubkey(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), isFixedVectorShuffle(), isUndefVector(), and performExtractsShuffleAction().

◆ isUndefVector< false >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< false >	(	const Value *	,
		const SmallBitVector &	)

◆ isUndefVector< true >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< true >	(	const Value *	,
		const SmallBitVector &	)

Referenced by performExtractsShuffleAction().

◆ isUsedOutsideBlock()

bool llvm::slpvectorizer::isUsedOutsideBlock ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all users are phi nodes or instructions from different blocks.

Definition at line 344 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and UsesLimit.

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ isValidForAlternation()

bool llvm::slpvectorizer::isValidForAlternation ( unsigned Opcode )

Returns: true if Opcode is allowed as part of the main/alternate instruction for SLP vectorization.

Example of unsupported opcode is SDIV that can potentially cause UB if the "shuffled out" lane would result in division by zero.

Definition at line 28 of file SLPCompatibilityAnalysis.cpp.

References llvm::Instruction::isIntDivRem().

Referenced by generateKeySubkey(), and getSameOpcode().

◆ isVectorLikeInstWithConstOps()

bool llvm::slpvectorizer::isVectorLikeInstWithConstOps ( Value * V )

Checks if V is one of vector-like instructions, i.e.

undef, insertelement/extractelement with constant indices for fixed vector type or extractvalue instruction.

Definition at line 38 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, llvm::isa(), and isConstant().

Referenced by allSameBlock(), generateKeySubkey(), getSameOpcode(), and tryToFindDuplicates().

◆ reorderScalars()

void llvm::slpvectorizer::reorderScalars	(	SmallVectorImpl< Value * > &	Scalars,
		ArrayRef< int >	Mask )

Reorders the list of scalars in accordance with the given Mask.

Definition at line 281 of file SLPUtils.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::PoisonValue::get(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::adjustExtracts(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::adjustExtracts(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ replicateMask()

SmallVector< Constant * > llvm::slpvectorizer::replicateMask	(	ArrayRef< Constant * >	Val,
		unsigned	VF )

Replicates the given Val VF times.

Definition at line 655 of file SLPUtils.cpp.

References assert(), llvm::CallingConv::C, llvm::enumerate(), I, llvm::none_of(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather().

◆ shortBundleName()

std::string llvm::slpvectorizer::shortBundleName	(	ArrayRef< Value * >	VL,
		int	Idx )

Print a short descriptor of the instruction bundle suitable for debug output.

Definition at line 75 of file SLPUtils.cpp.

References llvm::ArrayRef< T >::front(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::processBuildVector().

◆ transformScalarShuffleIndiciesToVector()

void llvm::slpvectorizer::transformScalarShuffleIndiciesToVector	(	unsigned	VecTyNumElements,
		SmallVectorImpl< int > &	Mask )

Definition at line 367 of file SLPUtils.cpp.

References llvm::enumerate(), I, llvm::MutableArrayRef(), llvm::PoisonMaskElem, and llvm::seq().

Variable Documentation

◆ UsesLimit

int llvm::slpvectorizer::UsesLimit = 64

inlineconstexpr

Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Definition at line 42 of file SLPUtils.h.

Referenced by slpvectorizer::BoUpSLP::buildExternalUses(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getTreeCost(), isCommutative(), and isUsedOutsideBlock().

Classes

Enumerations

Functions

Variables

Detailed Description

Enumeration Type Documentation

◆ UseMask

Function Documentation

◆ addMask()

◆ allConstant()

◆ allSameBlock()

◆ allSameOpcode()

◆ allSameType()

◆ areAllOperandsNonInsts()

◆ buildUseMask()

◆ calculateShufflevectorMask()

◆ doesInTreeUserNeedToExtract()

◆ doesNotNeedToBeScheduled()

◆ doesNotNeedToSchedule()

◆ fixupOrderingIndices()

◆ getAltInstrMask()

◆ getBlendedLoadCost()

◆ getElementIndex()

◆ getExtractIndex()

◆ getGEPCosts()

◆ getInsertExtractIndex()

◆ getInsertExtractIndex< ExtractElementInst >()

◆ getInsertExtractIndex< InsertElementInst >()

◆ getLocation()

◆ getMaskedDivRemIntrinsic()

◆ getNumberOfPotentiallyCommutativeOps()

◆ getNumElements()

◆ getNumElems()

◆ getPartNumElems()

◆ getShuffleCost()

◆ getShufflevectorNumGroups()

◆ inversePermutation()

◆ isCommutableOperand()

◆ isCommutative() [1/2]

◆ isCommutative() [2/2]

◆ isConstant()

◆ isEquivalentIntrinsicID()

◆ isSelectedBaseLoad()

◆ isSimple()

◆ isSplat()

◆ isUndefVector()

◆ isUndefVector< false >()

◆ isUndefVector< true >()

◆ isUsedOutsideBlock()

◆ isValidForAlternation()

◆ isVectorLikeInstWithConstOps()

◆ reorderScalars()

◆ replicateMask()

◆ shortBundleName()

◆ transformScalarShuffleIndiciesToVector()

Variable Documentation

◆ UsesLimit