A private "module" namespace for types and utilities used by this pass. More...

Classes
class	BinOpSameOpcodeHelper
	Helper class that determines VL can use the same opcode. More...
class	BoUpSLP
	Bottom Up SLP Vectorizer. More...
class	InstructionsState
	Main data required for vectorization of instructions. More...

Enumerations
enum class	UseMask { FirstArg , SecondArg , UndefsAsMask }
	Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask. More...

Functions
bool	isValidForAlternation (unsigned Opcode)
bool	isAbsorbableFMulOrFAdd (ArrayRef< Value * > VL, Value *V)
	Checks if `V` is a single-use fmul/fadd with operands outside `VL`.
bool	isAbsorbableCopyableFMulOrFAdd (const InstructionsState &S, Value *V)
	Checks if `V` is a copyable single-use fmul/fadd, absorbable as fmuladd(a, b, -0.0) or fmuladd(1.0, a, b).
bool	hasOnlyAbsorbableCopyableFMulOrFAdds (ArrayRef< Value * > VL)
	Checks if every copyable in `VL` is an absorbable fmul/fadd: the binops die instead of being computed and gathered.
static Instruction *	findInstructionWithOpcode (ArrayRef< Value * > VL, unsigned Opcode)
	Find an instruction with a specific opcode in VL.
static bool	areCompatibleCmpOps (Value BaseOp0, Value BaseOp1, Value Op0, Value Op1, const TargetLibraryInfo &TLI)
	Checks if the provided operands of 2 cmp instructions are compatible, i.e.
static bool	isCmpSameOrSwapped (const CmpInst BaseCI, const CmpInst CI, const TargetLibraryInfo &TLI)
InstructionsState	getSameOpcode (ArrayRef< Value * > VL, const TargetLibraryInfo &TLI)
std::pair< Instruction , SmallVector< Value > >	convertTo (Instruction *I, const InstructionsState &S)
bool	isAlternateInstruction (Instruction I, Instruction MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction `I` is an alternate operation for the given `MainOp` and `AltOp` instructions.
SmallVector< SmallVector< Value * > >	scanAltAssociativeOperands (const InstructionsState &S, const TargetLibraryInfo &TLI, ArrayRef< Value * > VL, ArrayRef< Value * > Op0, ArrayRef< Value * > Op1, SmallVectorImpl< Value * > &ReassocScalars, SmallBitVector &SubLanes)
	Peel the per-lane associative chains of an alternate node into operand columns.
InstructionCost	getShuffleCost (const TargetTransformInfo &TTI, TargetTransformInfo::ShuffleKind Kind, VectorType Tp, ArrayRef< int > Mask={}, TargetTransformInfo::TargetCostKind CostKind=TargetTransformInfo::TCK_RecipThroughput, int Index=0, VectorType SubTp=nullptr, ArrayRef< const Value * > Args={})
	Returns the cost of the shuffle instructions with the given `Kind`, vector type `Tp` and optional `Mask`.
std::pair< InstructionCost, InstructionCost >	getGEPCosts (const TargetTransformInfo &TTI, ArrayRef< Value * > Ptrs, Value BasePtr, unsigned Opcode, TargetTransformInfo::TargetCostKind CostKind, Type ScalarTy, VectorType *VecTy)
	Calculate the scalar and the vector costs from vectorizing set of GEPs.
InstructionCost	getBlendedLoadCost (const TargetTransformInfo &TTI, Type *VecTy, Align Alignment, unsigned AddressSpace, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of a BlendedLoadVectorize node loading `VecTy:` two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.
bool	isConstant (Value *V)
bool	isBinOpIdentityConstant (const Value *V, unsigned Opcode)
unsigned	getReassocCombineOpcode (unsigned Opcode)
bool	isReassocChainLink (const Instruction *I)
bool	isVectorLikeInstWithConstOps (Value *V)
	Checks if `V` is one of vector-like instructions, i.e.
unsigned	getNumElements (Type *Ty)
unsigned	getPartNumElems (unsigned Size, unsigned NumParts)
	Returns power-of-2 number of elements in a single register (part), given the total number of elements `Size` and number of registers (parts) `NumParts`.
unsigned	getNumElems (unsigned Size, unsigned PartNumElems, unsigned Part)
	Returns correct remaining number of elements, considering total amount `Size`, (power-of-2 number) of elements in a single register `PartNumElems` and current register (part) `Part`.
std::string	shortBundleName (ArrayRef< Value * > VL, int Idx=-1)
	Print a short descriptor of the instruction bundle suitable for debug output.
bool	allSameBlock (ArrayRef< Value * > VL)
bool	allConstant (ArrayRef< Value * > VL)
bool	isSplat (ArrayRef< Value * > VL)
Intrinsic::ID	isEquivalentIntrinsicID (Intrinsic::ID LHS, Intrinsic::ID RHS)
	Checks if `LHS` and `RHS` are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.
bool	isCommutative (const Instruction I, const Value ValWithUses, bool IsCopyable)
bool	isCommutative (const Instruction *I)
	This is a helper function to check whether `I` is commutative.
bool	isCommutableOperand (const Instruction I, Value ValWithUses, unsigned Op, bool IsCopyable=false)
	Checks if the operand is commutative.
unsigned	getNumberOfPotentiallyCommutativeOps (Instruction *I)
std::optional< unsigned >	getElementIndex (const Value *Inst, unsigned Offset)
bool	allSameOpcode (ArrayRef< Value * > VL)
std::optional< unsigned >	getExtractIndex (const Instruction *E)
void	inversePermutation (ArrayRef< unsigned > Indices, SmallVectorImpl< int > &Mask)
	Compute the inverse permutation `Mask` of `Indices`.
void	reorderScalars (SmallVectorImpl< Value * > &Scalars, ArrayRef< int > Mask)
	Reorders the list of scalars in accordance with the given `Mask`.
bool	allSameType (ArrayRef< Value * > VL)
template<typename T>
std::optional< unsigned >	getInsertExtractIndex (const Value *Inst, unsigned Offset)
template std::optional< unsigned >	getInsertExtractIndex< InsertElementInst > (const Value *, unsigned)
template std::optional< unsigned >	getInsertExtractIndex< ExtractElementInst > (const Value *, unsigned)
bool	areAllOperandsNonInsts (Value *V)
	Checks if the provided value does not require scheduling.
bool	isUsedOutsideBlock (Value *V)
	Checks if the provided value does not require scheduling.
bool	doesNotNeedToBeScheduled (Value *V)
	Checks if the specified value does not require scheduling.
bool	doesNotNeedToSchedule (ArrayRef< Value * > VL)
	Checks if the specified array of instructions does not require scheduling.
void	transformScalarShuffleIndiciesToVector (unsigned VecTyNumElements, SmallVectorImpl< int > &Mask)
unsigned	getShufflevectorNumGroups (ArrayRef< Value * > VL)
SmallVector< int >	calculateShufflevectorMask (ArrayRef< Value * > VL)
SmallBitVector	buildUseMask (int VF, ArrayRef< int > Mask, UseMask MaskArg)
	Prepares a use bitset for the given mask either for the first argument or for the second.
template<bool IsPoisonOnly>
SmallBitVector	isUndefVector (const Value *V, const SmallBitVector &UseMask={})
	Checks if the given value is actually an undefined constant vector.
template SmallBitVector	isUndefVector< false > (const Value *, const SmallBitVector &)
template SmallBitVector	isUndefVector< true > (const Value *, const SmallBitVector &)
bool	doesInTreeUserNeedToExtract (Value Scalar, Instruction UserInst, TargetLibraryInfo TLI, const TargetTransformInfo TTI)
MemoryLocation	getLocation (Instruction *I)
bool	isSimple (Instruction *I)
bool	isSelectedBaseLoad (Type ScalarTy, ArrayRef< Value > PointerOps, const DataLayout &DL, Value &TrueBase, Value &FalseBase, SmallVectorImpl< Value * > &Conditions)
	Checks if the loads with scalar type `ScalarTy` and pointer operands `PointerOps` are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?
void	addMask (SmallVectorImpl< int > &Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs=false)
	Shuffles `Mask` in accordance with the given `SubMask`.
void	fixupOrderingIndices (MutableArrayRef< unsigned > Order)
	Order may have elements assigned special value (size) which is out of bounds.
SmallBitVector	getAltInstrMask (ArrayRef< Value * > VL, Type *ScalarTy, unsigned Opcode0, unsigned Opcode1)
SmallVector< Constant * >	replicateMask (ArrayRef< Constant * > Val, unsigned VF)
	Replicates the given `Val` `VF` times.
Intrinsic::ID	getMaskedDivRemIntrinsic (unsigned Opcode)
static bool	isNonProfitableIndex (const Instruction *I)
	Returns true if `I` is a part of a single-use chain, computing an address, which does not pay off the vectorization: a constant table is accessed by a gather, while the indices, unrelated between the lanes, require a full buildvector, unlike the ones, shifted by a constant from a common base.
bool	isOnceUsedSeed (const Instruction *I)
	Returns true if `I` forms a vectorizable bundle on its own and its single user does not tear the vector apart.
Instruction *	lookThroughCastRoundTrip (Value *V, bool MustBeElidable)
	If `V` is a single-use fpext of a single-use fptrunc forming a round-trip back to the type of `V`, returns the fptrunc; the round-trip source is its operand, always an instruction of the same type as `V`.

Variables
constexpr int	UsesLimit = 64
	Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Detailed Description

A private "module" namespace for types and utilities used by this pass.

These are implementation details and should not be used by clients.

Enumeration Type Documentation

◆ UseMask

enum class llvm::slpvectorizer::UseMask

strong

Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask.

Enumerator

FirstArg

The mask is expected to be for permutation of 1-2 vectors, check for the mask elements for the first argument (mask indices are in range [0:VF)).

SecondArg

The mask is expected to be for permutation of 2 vectors, check for the mask elements for the second argument (mask indices are in range [VF:2*VF))

UndefsAsMask

Consider undef mask elements (-1) as placeholders for future shuffle elements and mark them as ones as being used in future.

Non-undef elements are considered as unused since they're already marked as used in the mask.

Definition at line 240 of file SLPUtils.h.

Function Documentation

◆ addMask()

void llvm::slpvectorizer::addMask	(	SmallVectorImpl< int > &	Mask,
		ArrayRef< int >	SubMask,
		bool	ExtendingManyInputs = false )

Shuffles Mask in accordance with the given SubMask.

Parameters

ExtendingManyInputs Supports reshuffling of the mask with not only one but two input vectors.

Definition at line 616 of file SLPUtils.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ allConstant()

bool llvm::slpvectorizer::allConstant ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are constants (but not globals/constant expressions).

Definition at line 131 of file SLPUtils.cpp.

References llvm::all_of(), and isConstant().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getSpillCost(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameBlock()

bool llvm::slpvectorizer::allSameBlock ( ArrayRef< Value * > VL )

Returns: True if all of the instructions in VL are in the same block.

Definition at line 109 of file SLPUtils.cpp.

References llvm::all_of(), llvm::cast(), llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), II, llvm::isa(), llvm::IsaPred, isVectorLikeInstWithConstOps(), and llvm::iterator_range().

Referenced by checkEVsForVecCalls(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameOpcode()

bool llvm::slpvectorizer::allSameOpcode ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL use the same opcode. For comparison instructions, also checks if predicates match. PoisonValues are considered matching. Interchangeable instructions are not considered.

Definition at line 254 of file SLPUtils.cpp.

References llvm::all_of(), llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::Instruction::getOpcode(), llvm::isa(), llvm::IsaPred, and llvm::make_range().

Referenced by getSameOpcode().

◆ allSameType()

bool llvm::slpvectorizer::allSameType ( ArrayRef< Value * > VL )

Returns: True iff every value in VL has the same Type as the first.

Definition at line 315 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::consume_front(), and llvm::ArrayRef< T >::empty().

Referenced by slpvectorizer::BoUpSLP::buildTree(), slpvectorizer::BoUpSLP::buildTree(), and slpvectorizer::BoUpSLP::getReorderingData().

◆ areAllOperandsNonInsts()

bool llvm::slpvectorizer::areAllOperandsNonInsts ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all operands are either not instructions or phi nodes or instructions from different blocks.

Definition at line 355 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and llvm::mayHaveNonDefUseDependency().

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ areCompatibleCmpOps()

bool llvm::slpvectorizer::areCompatibleCmpOps	(	Value *	BaseOp0,
		Value *	BaseOp1,
		Value *	Op0,
		Value *	Op1,
		const TargetLibraryInfo &	TLI )

static

Checks if the provided operands of 2 cmp instructions are compatible, i.e.

compatible instructions or constants, or just some other regular values.

Definition at line 533 of file SLPCompatibilityAnalysis.cpp.

References getSameOpcode(), llvm::isa(), and isConstant().

Referenced by isCmpSameOrSwapped().

◆ buildUseMask()

SmallBitVector llvm::slpvectorizer::buildUseMask	(	int	VF,
		ArrayRef< int >	Mask,
		UseMask	MaskArg )

Prepares a use bitset for the given mask either for the first argument or for the second.

Definition at line 463 of file SLPUtils.cpp.

References llvm::enumerate(), FirstArg, llvm::PoisonMaskElem, SecondArg, and UndefsAsMask.

Referenced by performExtractsShuffleAction().

◆ calculateShufflevectorMask()

SmallVector< int > llvm::slpvectorizer::calculateShufflevectorMask ( ArrayRef< Value * > VL )

Returns: a shufflevector mask which is used to vectorize shufflevectors e.g., %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> the result is <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>

Definition at line 446 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::ArrayRef< T >::front(), getShufflevectorNumGroups(), and llvm::PoisonMaskElem.

◆ convertTo()

std::pair< Instruction , SmallVector< Value > > llvm::slpvectorizer::convertTo	(	Instruction *	I,
		const InstructionsState &	S )

Returns: the main or alternate operation from S matching I, together with the operands of I adjusted to the selected operation.

Definition at line 777 of file SLPCompatibilityAnalysis.cpp.

References assert(), Converter, llvm::dyn_cast(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), and I.

◆ doesInTreeUserNeedToExtract()

bool llvm::slpvectorizer::doesInTreeUserNeedToExtract	(	Value *	Scalar,
		Instruction *	UserInst,
		TargetLibraryInfo *	TLI,
		const TargetTransformInfo *	TTI )

Returns: True if in-tree use also needs extract. This refers to possible scalar operand in vectorized instruction.

Definition at line 530 of file SLPUtils.cpp.

References llvm::any_of(), llvm::CallBase::args(), llvm::cast(), llvm::enumerate(), llvm::Instruction::getOpcode(), llvm::LoadInst::getPointerOperand(), and llvm::getVectorIntrinsicIDForCall().

Referenced by slpvectorizer::BoUpSLP::buildExternalUses().

◆ doesNotNeedToBeScheduled()

bool llvm::slpvectorizer::doesNotNeedToBeScheduled ( Value * V )

Checks if the specified value does not require scheduling.

It does not require scheduling if all operands and all users do not need to be scheduled in the current basic block.

Definition at line 382 of file SLPUtils.cpp.

References areAllOperandsNonInsts(), and isUsedOutsideBlock().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and llvm::slpvectorizer::InstructionsState::isNonSchedulable().

◆ doesNotNeedToSchedule()

bool llvm::slpvectorizer::doesNotNeedToSchedule ( ArrayRef< Value * > VL )

Checks if the specified array of instructions does not require scheduling.

It is so if all either instructions have operands that do not require scheduling or their users do not require scheduling since they are phis or in other basic blocks.

Definition at line 386 of file SLPUtils.cpp.

References llvm::all_of(), areAllOperandsNonInsts(), llvm::ArrayRef< T >::empty(), and isUsedOutsideBlock().

Referenced by tryToFindDuplicates().

◆ findInstructionWithOpcode()

Instruction * llvm::slpvectorizer::findInstructionWithOpcode	(	ArrayRef< Value * >	VL,
		unsigned	Opcode )

static

Find an instruction with a specific opcode in VL.

Parameters

VL	Array of values to search through. Must contain only Instructions and PoisonValues.
Opcode	The instruction opcode to search for

Returns

The first instruction found with matching opcode
nullptr if no matching instruction is found

Definition at line 518 of file SLPCompatibilityAnalysis.cpp.

References assert(), llvm::cast(), and llvm::isa().

Referenced by getSameOpcode().

◆ fixupOrderingIndices()

void llvm::slpvectorizer::fixupOrderingIndices ( MutableArrayRef< unsigned > Order )

Order may have elements assigned special value (size) which is out of bounds.

Such indices only appear on places which correspond to undef values (see canReuseExtract for details) and used in order to avoid undef values have effect on operands ordering. The first loop below simply finds all unused indices and then the next loop nest assigns these indices for undef values positions. As an example below Order has two undef positions and they have assigned values 3 and 7 respectively: before: 6 9 5 4 9 2 1 0 after: 6 3 5 4 7 2 1 0

Definition at line 641 of file SLPUtils.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::SmallBitVector::find_first(), llvm::SmallBitVector::find_next(), I, llvm::SmallBitVector::none(), llvm::SmallBitVector::reset(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getAltInstrMask()

SmallBitVector llvm::slpvectorizer::getAltInstrMask	(	ArrayRef< Value * >	VL,
		Type *	ScalarTy,
		unsigned	Opcode0,
		unsigned	Opcode1 )

Returns: a bitset for selecting opcodes. false for Opcode0 and true for Opcode1.

Definition at line 665 of file SLPUtils.cpp.

References llvm::cast(), getNumElements(), llvm::isa(), llvm::seq(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getBlendedLoadCost()

InstructionCost llvm::slpvectorizer::getBlendedLoadCost	(	const TargetTransformInfo &	TTI,
		Type *	VecTy,
		Align	Alignment,
		unsigned	AddressSpace,
		TargetTransformInfo::TargetCostKind	CostKind )

Returns the cost of a BlendedLoadVectorize node loading VecTy: two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.

The blend mask is a separate operand node, so its cost is counted there, not here.

Definition at line 131 of file SLPCostAnalysis.cpp.

References llvm::CmpInst::BAD_ICMP_PREDICATE, CostKind, and llvm::CmpInst::makeCmpResultType().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ getElementIndex()

std::optional< unsigned > llvm::slpvectorizer::getElementIndex	(	const Value *	Inst,
		unsigned	Offset = 0 )

Returns: inserting or extracting index of InsertElement, ExtractElement or InsertValue instruction, using Offset as base offset for index.; std::nullopt if the index is not an immediate.

Definition at line 226 of file SLPUtils.cpp.

References llvm::dyn_cast(), getInsertExtractIndex(), I, IV, and llvm::Offset.

Referenced by areTwoInsertFromSameBuildVector(), slpvectorizer::BoUpSLP::getReorderingData(), and isUndefVector().

◆ getExtractIndex()

std::optional< unsigned > llvm::slpvectorizer::getExtractIndex ( const Instruction * E )

Returns: Optional element Idx for Extract{Value,Element} instructions.

Definition at line 272 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), and getNumElements().

Referenced by slpvectorizer::BoUpSLP::getReorderingData().

◆ getGEPCosts()

std::pair< InstructionCost, InstructionCost > llvm::slpvectorizer::getGEPCosts	(	const TargetTransformInfo &	TTI,
		ArrayRef< Value * >	Ptrs,
		Value *	BasePtr,
		unsigned	Opcode,
		TTI::TargetCostKind	CostKind,
		Type *	ScalarTy,
		VectorType *	VecTy )

Calculate the scalar and the vector costs from vectorizing set of GEPs.

Definition at line 51 of file SLPCostAnalysis.cpp.

References llvm::all_of(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::IsaPred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::TargetTransformInfo::TCC_Free.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

◆ getInsertExtractIndex()

template<typename T>

std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex	(	const Value *	Inst,
		unsigned	Offset )

Returns: inserting or extracting index of InsertElement / ExtractElement instruction, using Offset as base offset for index. Only instantiated for InsertElementInst and ExtractElementInst (see SLPUtils.cpp).

Definition at line 322 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::Offset.

Referenced by getElementIndex().

◆ getInsertExtractIndex< ExtractElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< ExtractElementInst >	(	const Value *	,
		unsigned	)

◆ getInsertExtractIndex< InsertElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< InsertElementInst >	(	const Value *	,
		unsigned	)

◆ getLocation()

MemoryLocation llvm::slpvectorizer::getLocation ( Instruction * I )

Returns: the AA location that is being access by the instruction.

Definition at line 558 of file SLPUtils.cpp.

References llvm::dyn_cast(), llvm::MemoryLocation::get(), and I.

◆ getMaskedDivRemIntrinsic()

Intrinsic::ID llvm::slpvectorizer::getMaskedDivRemIntrinsic ( unsigned Opcode )

Returns: the masked division/remainder intrinsic corresponding to Opcode. Disabled lanes of these intrinsics are poison rather than UB, unlike the plain opcode.

Definition at line 688 of file SLPUtils.cpp.

References llvm_unreachable.

Referenced by getMaskedDivRemCost().

◆ getNumberOfPotentiallyCommutativeOps()

unsigned llvm::slpvectorizer::getNumberOfPotentiallyCommutativeOps ( Instruction * I )

Returns: number of operands of I, considering commutativity. Returns 2 for commutative intrinsics.

Parameters

I	The instruction to check for commutativity

Definition at line 216 of file SLPUtils.cpp.

References I, llvm::isa(), and isCommutative().

◆ getNumElements()

unsigned llvm::slpvectorizer::getNumElements ( Type * Ty )

Returns: the number of elements for Ty.

Definition at line 82 of file SLPUtils.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::getVectorizedTypeVF(), llvm::isa(), and llvm::isVectorizedTy().

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate(), getAltInstrMask(), getExtractIndex(), getExtractWithExtendCost(), getNumberOfParts(), getVectorCallCosts(), getWidenedType(), and slpvectorizer::BoUpSLP::vectorizeTree().

◆ getNumElems()

unsigned llvm::slpvectorizer::getNumElems	(	unsigned	Size,
		unsigned	PartNumElems,
		unsigned	Part )

Returns correct remaining number of elements, considering total amount Size, (power-of-2 number) of elements in a single register PartNumElems and current register (part) Part.

Definition at line 94 of file SLPUtils.cpp.

References Size.

Referenced by slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getPartNumElems()

unsigned llvm::slpvectorizer::getPartNumElems	(	unsigned	Size,
		unsigned	NumParts )

Returns power-of-2 number of elements in a single register (part), given the total number of elements Size and number of registers (parts) NumParts.

Definition at line 90 of file SLPUtils.cpp.

References llvm::bit_ceil(), llvm::divideCeil(), and Size.

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getReassocCombineOpcode()

unsigned llvm::slpvectorizer::getReassocCombineOpcode ( unsigned Opcode )

Returns: the opcode of the combines emitted for a reassociated node: subtract chains regroup their positive and negative operand columns with plain adds.

Definition at line 43 of file SLPUtils.cpp.

Referenced by getReassocColumnsQuality(), and scanAssociativeOperands().

◆ getSameOpcode()

InstructionsState llvm::slpvectorizer::getSameOpcode	(	ArrayRef< Value * >	VL,
		const TargetLibraryInfo &	TLI )

Returns: analysis of the Instructions in VL described in InstructionsState, the Opcode that we suppose the whole list could be vectorized even if its structure is diverse.

Definition at line 566 of file SLPCompatibilityAnalysis.cpp.

References llvm::slpvectorizer::BinOpSameOpcodeHelper::add(), llvm::all_of(), allSameOpcode(), assert(), llvm::CmpInst::BAD_ICMP_PREDICATE, Call, llvm::cast(), llvm::SetVector< T, Vector, Set, N >::contains(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::find_if(), findInstructionWithOpcode(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::slpvectorizer::BinOpSameOpcodeHelper::getAltOpcode(), llvm::CallBase::getBundleOperandsStartIndex(), llvm::CallBase::getCalledFunction(), llvm::slpvectorizer::BinOpSameOpcodeHelper::getMainOpcode(), llvm::VFDatabase::getMappings(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::getVectorIntrinsicIDForCall(), llvm::slpvectorizer::BinOpSameOpcodeHelper::hasDefinedAltOpcode(), llvm::slpvectorizer::BinOpSameOpcodeHelper::hasDefinedMainOpcode(), llvm::CallBase::hasOperandBundles(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::slpvectorizer::InstructionsState::invalid(), llvm::isa(), llvm::IsaPred, isCmpSameOrSwapped(), isEquivalentIntrinsicID(), llvm::isTriviallyVectorizable(), isValidForAlternation(), isVectorLikeInstWithConstOps(), llvm::iterator_range(), llvm::Intrinsic::not_intrinsic, llvm::User::op_begin(), llvm::ArrayRef< T >::size(), llvm::SetVector< T, Vector, Set, N >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by areCompatibleCmpOps(), arePointersCompatible(), canConvertToFMA(), checkEVsForVecCalls(), compareCmp(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::VLOperands::reorder().

◆ getShuffleCost()

InstructionCost llvm::slpvectorizer::getShuffleCost	(	const TargetTransformInfo &	TTI,
		TargetTransformInfo::ShuffleKind	Kind,
		VectorType *	Tp,
		ArrayRef< int >	Mask = {},
		TargetTransformInfo::TargetCostKind	CostKind = TargetTransformInfo::TCK_RecipThroughput,
		int	Index = 0,
		VectorType *	SubTp = nullptr,
		ArrayRef< const Value * >	Args = {} )

Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.

Adds SLP-specific cost estimation for insert subvector pattern.

Definition at line 25 of file SLPCostAnalysis.cpp.

References CostKind, llvm::FixedVectorType::get(), llvm::VectorType::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::Type::getScalarType(), llvm::ShuffleVectorInst::isInsertSubvectorMask(), llvm::TargetTransformInfo::SK_InsertSubvector, llvm::TargetTransformInfo::SK_PermuteTwoSrc, and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canVectorizeLoads(), getExtractWithExtendCost(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getTreeCost(), getVectorInstrCost(), isMaskedLoadCompress(), and tryToFindDuplicates().

◆ getShufflevectorNumGroups()

unsigned llvm::slpvectorizer::getShufflevectorNumGroups ( ArrayRef< Value * > VL )

Returns

the number of groups of shufflevector A group has the following features

All of value in a group are shufflevector.
The mask of all shufflevector is isExtractSubvectorMask.
The mask of all shufflevector uses all of the elements of the source. e.g., it is 1 group (%0) %1 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %2 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> it is 2 groups (%3 and %4) %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> it is 0 group %12 = shufflevector <8 x i16> %10, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %13 = shufflevector <8 x i16> %11, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>

Definition at line 406 of file SLPUtils.cpp.

References llvm::SmallBitVector::all(), llvm::all_of(), assert(), llvm::cast(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), I, llvm::IsaPred, llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().

Referenced by calculateShufflevectorMask().

◆ hasOnlyAbsorbableCopyableFMulOrFAdds()

bool llvm::slpvectorizer::hasOnlyAbsorbableCopyableFMulOrFAdds ( ArrayRef< Value * > VL )

Checks if every copyable in VL is an absorbable fmul/fadd: the binops die instead of being computed and gathered.

Operand order is normalized when the operands are built.

Definition at line 426 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, llvm::isa(), isAbsorbableFMulOrFAdd(), and llvm::RecurrenceDescriptor::isFMulAddIntrinsic().

◆ inversePermutation()

void llvm::slpvectorizer::inversePermutation	(	ArrayRef< unsigned >	Indices,
		SmallVectorImpl< int > &	Mask )

Compute the inverse permutation Mask of Indices.

Definition at line 296 of file SLPUtils.cpp.

References I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::addOrdered(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getReorderingData(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ isAbsorbableCopyableFMulOrFAdd()

bool llvm::slpvectorizer::isAbsorbableCopyableFMulOrFAdd	(	const InstructionsState &	S,
		Value *	V )

Checks if V is a copyable single-use fmul/fadd, absorbable as fmuladd(a, b, -0.0) or fmuladd(1.0, a, b).

Definition at line 418 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, and llvm::slpvectorizer::InstructionsState::isCopyableElement().

◆ isAbsorbableFMulOrFAdd()

bool llvm::slpvectorizer::isAbsorbableFMulOrFAdd	(	ArrayRef< Value * >	VL,
		Value *	V )

Checks if V is a single-use fmul/fadd with operands outside VL.

Definition at line 408 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, and llvm::none_of().

Referenced by hasOnlyAbsorbableCopyableFMulOrFAdds().

◆ isAlternateInstruction()

bool llvm::slpvectorizer::isAlternateInstruction	(	Instruction *	I,
		Instruction *	MainOp,
		Instruction *	AltOp,
		const TargetLibraryInfo &	TLI )

Checks if the specified instruction I is an alternate operation for the given MainOp and AltOp instructions.

Definition at line 790 of file SLPCompatibilityAnalysis.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), llvm::CmpInst::getSwappedPredicate(), I, isCmpSameOrSwapped(), and P.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getReorderingData(), and scanAltAssociativeOperands().

◆ isBinOpIdentityConstant()

bool llvm::slpvectorizer::isBinOpIdentityConstant	(	const Value *	V,
		unsigned	Opcode )

Returns: True if V is the integer identity constant for binary Opcode (e.g. 0 for add, 1 for mul, all-ones for and). Floating-point identities are excluded: a ConstantInt never matches the ConstantFP getBinOpIdentity() returns for FAdd/FMul, whose identity fast-math may break anyway.

Definition at line 38 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::ConstantExpr::getBinOpIdentity().

Referenced by getReassocColumnsQuality().

◆ isCmpSameOrSwapped()

bool llvm::slpvectorizer::isCmpSameOrSwapped	(	const CmpInst *	BaseCI,
		const CmpInst *	CI,
		const TargetLibraryInfo &	TLI )

static

Returns: true if a compare instruction CI has similar "look" and same predicate as BaseCI, "as is" or with its operands and predicate swapped, false otherwise.

Definition at line 547 of file SLPCompatibilityAnalysis.cpp.

References areCompatibleCmpOps(), assert(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::CmpInst::getSwappedPredicate(), and llvm::Value::getType().

Referenced by getSameOpcode(), and isAlternateInstruction().

◆ isCommutableOperand()

bool llvm::slpvectorizer::isCommutableOperand	(	const Instruction *	I,
		Value *	ValWithUses,
		unsigned	Op,
		bool	IsCopyable = false )

Checks if the operand is commutative.

In commutative operations, not all operands might commutable, e.g. for fmuladd only 2 first operands are commutable.

Definition at line 198 of file SLPUtils.cpp.

References assert(), llvm::dyn_cast(), I, llvm::isa(), and isCommutative().

◆ isCommutative() [1/2]

bool llvm::slpvectorizer::isCommutative ( const Instruction * I )

This is a helper function to check whether I is commutative.

This is a convenience wrapper that calls the two-parameter version of isCommutative with the same instruction for both parameters. This is the common case where the instruction being checked for commutativity is the same as the instruction whose uses are analyzed for special patterns (see the two-parameter version above for details).

Parameters

I	The instruction to check for commutativity

Returns: true if the instruction is commutative, false otherwise

Definition at line 196 of file SLPUtils.cpp.

References I, and isCommutative().

◆ isCommutative() [2/2]

bool llvm::slpvectorizer::isCommutative	(	const Instruction *	I,
		const Value *	ValWithUses,
		bool	IsCopyable = false )

Returns: True if I is commutative, handles CmpInst and BinaryOperator. For BinaryOperator, it also checks if ValWithUses is used in specific patterns that make it effectively commutative (like equality comparisons with zero). In most cases, users should not call this function directly (since I and ValWithUses are the same). However, when analyzing interchangeable instructions, we need to use the converted opcode along with the original uses.

Parameters

I	The instruction to check for commutativity
ValWithUses	The value whose uses are analyzed for special patterns

Definition at line 161 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::Value::hasNUsesOrMore(), llvm::Value::hasUseList(), I, llvm::Value::uses(), and UsesLimit.

Referenced by getNumberOfPotentiallyCommutativeOps(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getScoreAtLevelRec(), isCommutableOperand(), isCommutative(), and slpvectorizer::BoUpSLP::isProfitableToReorder().

◆ isConstant()

bool llvm::slpvectorizer::isConstant ( Value * V )

Returns: True if the value is a constant (but not globals/constant expressions).

Definition at line 34 of file SLPUtils.cpp.

References llvm::isa().

Referenced by allConstant(), areCompatibleCmpOps(), arePointersCompatible(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and isVectorLikeInstWithConstOps().

◆ isEquivalentIntrinsicID()

Intrinsic::ID llvm::slpvectorizer::isEquivalentIntrinsicID	(	Intrinsic::ID	LHS,
		Intrinsic::ID	RHS )

Checks if LHS and RHS are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.

Returns: the intrinsic ID to use for the pair (RHS if the IDs match, otherwise Intrinsic::fma), or Intrinsic::not_intrinsic if they are not equivalent.

Definition at line 152 of file SLPUtils.cpp.

References llvm::Intrinsic::not_intrinsic.

Referenced by getSameOpcode(), and llvm::slpvectorizer::InstructionsState::isSameOperation().

◆ isNonProfitableIndex()

bool llvm::slpvectorizer::isNonProfitableIndex ( const Instruction * I )

static

Returns true if I is a part of a single-use chain, computing an address, which does not pay off the vectorization: a constant table is accessed by a gather, while the indices, unrelated between the lanes, require a full buildvector, unlike the ones, shifted by a constant from a common base.

Definition at line 707 of file SLPUtils.cpp.

References _, llvm::dyn_cast(), GEP, I, llvm::isa(), llvm::none_of(), and llvm::seq().

Referenced by isOnceUsedSeed().

◆ isOnceUsedSeed()

bool llvm::slpvectorizer::isOnceUsedSeed ( const Instruction * I )

Returns true if I forms a vectorizable bundle on its own and its single user does not tear the vector apart.

Loads and addresses are excluded: the tree is built without the users, so it does not pay off the extracts. A cast, feeding a multi-used cast, is excluded for the same reason, such a user stays scalar. The fp-to-int conversions move the result to the other register domain, so the extracts are paid on top of the repacking. The values, feeding the inserts, are vectorized together with them by the dedicated attempt.

Definition at line 730 of file SLPUtils.cpp.

References C(), llvm::dyn_cast(), llvm::ConstantExpr::getBinOpAbsorber(), llvm::ConstantExpr::getBinOpIdentity(), I, llvm::isa(), isNonProfitableIndex(), and llvm::seq().

◆ isReassocChainLink()

bool llvm::slpvectorizer::isReassocChainLink ( const Instruction * I )

Returns: True if I can be a link of a flattenable binary chain: subtracts flatten as adds of a negated leaf, float subtracts need reassoc to allow the regrouping.

Definition at line 54 of file SLPUtils.cpp.

References I.

Referenced by scanAltAssociativeOperands(), and scanAssociativeOperands().

◆ isSelectedBaseLoad()

bool llvm::slpvectorizer::isSelectedBaseLoad	(	Type *	ScalarTy,
		ArrayRef< Value * >	PointerOps,
		const DataLayout &	DL,
		Value *&	TrueBase,
		Value *&	FalseBase,
		SmallVectorImpl< Value * > &	Conditions )

Checks if the loads with scalar type ScalarTy and pointer operands PointerOps are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?

A[i] : B[i] takes. On success TrueBase / FalseBase are the candidate bases and Conditions holds each lane's select condition, used to build the blend mask. Lane Idx must be at Base + Idx * sizeof(ScalarTy); only dense, natural lane order starting at the base is recognized (reordered or partial groups fall back to Gather/Scatter).

Definition at line 576 of file SLPUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::sampleprof::Base, DL, llvm::dyn_cast(), llvm::enumerate(), F, GEP, llvm::APInt::getZExtValue(), llvm::APInt::isNegative(), llvm::Offset, P, llvm::ArrayRef< T >::size(), and T.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ isSimple()

bool llvm::slpvectorizer::isSimple ( Instruction * I )

Returns: True if the instruction is not a volatile or atomic load/store.

Definition at line 566 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, and MI.

◆ isSplat()

bool llvm::slpvectorizer::isSplat ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are identical or some of them are UndefValue.

Definition at line 137 of file SLPUtils.cpp.

References llvm::isa().

Referenced by llvm::DOTGraphTraits< BoUpSLP * >::getNodeLabel(), and slpvectorizer::BoUpSLP::getReorderingData().

◆ isUndefVector()

template<bool IsPoisonOnly>

SmallBitVector llvm::slpvectorizer::isUndefVector	(	const Value *	V,
		const SmallBitVector &	UseMask = {} )

Checks if the given value is actually an undefined constant vector.

Also, if the UseMask is not empty, tries to check if the non-masked elements actually mask the insertelement buildvector, if any.

Definition at line 480 of file SLPUtils.cpp.

References llvm::sampleprof::Base, C(), llvm::dyn_cast(), getElementIndex(), I, II, llvm::isa(), isUndefVector(), llvm::SmallBitVector::reset(), and T.

Referenced by generateKeySubkey(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), isFixedVectorShuffle(), isUndefVector(), and performExtractsShuffleAction().

◆ isUndefVector< false >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< false >	(	const Value *	,
		const SmallBitVector &	)

◆ isUndefVector< true >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< true >	(	const Value *	,
		const SmallBitVector &	)

Referenced by performExtractsShuffleAction().

◆ isUsedOutsideBlock()

bool llvm::slpvectorizer::isUsedOutsideBlock ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all users are phi nodes or instructions from different blocks.

Definition at line 368 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and UsesLimit.

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ isValidForAlternation()

bool llvm::slpvectorizer::isValidForAlternation ( unsigned Opcode )

Returns: true if Opcode is allowed as part of the main/alternate instruction for SLP vectorization.

Example of unsupported opcode is SDIV that can potentially cause UB if the "shuffled out" lane would result in division by zero.

Definition at line 40 of file SLPCompatibilityAnalysis.cpp.

References llvm::Instruction::isIntDivRem().

Referenced by generateKeySubkey(), and getSameOpcode().

◆ isVectorLikeInstWithConstOps()

bool llvm::slpvectorizer::isVectorLikeInstWithConstOps ( Value * V )

Checks if V is one of vector-like instructions, i.e.

undef, insertelement/extractelement with constant indices for fixed vector type or extractvalue instruction.

Definition at line 62 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, llvm::isa(), and isConstant().

Referenced by allSameBlock(), generateKeySubkey(), getSameOpcode(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::slpvectorizer::InstructionsState::isNonSchedulable(), and tryToFindDuplicates().

◆ lookThroughCastRoundTrip()

Instruction * llvm::slpvectorizer::lookThroughCastRoundTrip	(	Value *	V,
		bool	MustBeElidable )

If V is a single-use fpext of a single-use fptrunc forming a round-trip back to the type of V, returns the fptrunc; the round-trip source is its operand, always an instruction of the same type as V.

If MustBeElidable, matches only when the intermediate rounding may be removed: both casts must allow contraction and the widening cast cannot produce nan/inf.

Definition at line 757 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::isa().

◆ reorderScalars()

void llvm::slpvectorizer::reorderScalars	(	SmallVectorImpl< Value * > &	Scalars,
		ArrayRef< int >	Mask )

Reorders the list of scalars in accordance with the given Mask.

Definition at line 305 of file SLPUtils.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::PoisonValue::get(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::adjustExtracts(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::adjustExtracts(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ replicateMask()

SmallVector< Constant * > llvm::slpvectorizer::replicateMask	(	ArrayRef< Constant * >	Val,
		unsigned	VF )

Replicates the given Val VF times.

Definition at line 679 of file SLPUtils.cpp.

References assert(), C(), llvm::enumerate(), I, llvm::none_of(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather().

◆ scanAltAssociativeOperands()

SmallVector< SmallVector< Value * > > llvm::slpvectorizer::scanAltAssociativeOperands	(	const InstructionsState &	S,
		const TargetLibraryInfo &	TLI,
		ArrayRef< Value * >	VL,
		ArrayRef< Value * >	Op0,
		ArrayRef< Value * >	Op1,
		SmallVectorImpl< Value * > &	ReassocScalars,
		SmallBitVector &	SubLanes )

Peel the per-lane associative chains of an alternate node into operand columns.

Lanes peel in lockstep and only chain links with the lane's own opcode, so every combine level keeps the root's main/alt opcode pattern and a subtract lane never becomes an add of a negated leaf. Only the leading (running) column peels: peeling a subtracted subtract would flip signs. SubLanes records the subtract lanes for the realignment sign query. Returns the flattened columns, empty when no level peels.

Definition at line 813 of file SLPCompatibilityAnalysis.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::slpvectorizer::InstructionsState::getAltOp(), llvm::slpvectorizer::InstructionsState::getAltOpcode(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::User::getOperand(), I, llvm::SmallVectorImpl< T >::insert(), isAlternateInstruction(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::Instruction::isCommutative(), isReassocChainLink(), llvm::map_to_vector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallBitVector::resize(), llvm::seq(), llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and std::swap().

◆ shortBundleName()

std::string llvm::slpvectorizer::shortBundleName	(	ArrayRef< Value * >	VL,
		int	Idx )

Print a short descriptor of the instruction bundle suitable for debug output.

Definition at line 99 of file SLPUtils.cpp.

References llvm::ArrayRef< T >::front(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::processBuildVector().

◆ transformScalarShuffleIndiciesToVector()

void llvm::slpvectorizer::transformScalarShuffleIndiciesToVector	(	unsigned	VecTyNumElements,
		SmallVectorImpl< int > &	Mask )

Definition at line 391 of file SLPUtils.cpp.

References llvm::enumerate(), I, llvm::MutableArrayRef(), llvm::PoisonMaskElem, and llvm::seq().

Variable Documentation

◆ UsesLimit

int llvm::slpvectorizer::UsesLimit = 64

inlineconstexpr

Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Definition at line 42 of file SLPUtils.h.

Referenced by slpvectorizer::BoUpSLP::buildExternalUses(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getTreeCost(), isCommutative(), and isUsedOutsideBlock().

Classes

Enumerations

Functions

Variables

Detailed Description

Enumeration Type Documentation

◆ UseMask

Function Documentation

◆ addMask()

◆ allConstant()

◆ allSameBlock()

◆ allSameOpcode()

◆ allSameType()

◆ areAllOperandsNonInsts()

◆ areCompatibleCmpOps()

◆ buildUseMask()

◆ calculateShufflevectorMask()

◆ convertTo()

◆ doesInTreeUserNeedToExtract()

◆ doesNotNeedToBeScheduled()

◆ doesNotNeedToSchedule()

◆ findInstructionWithOpcode()

◆ fixupOrderingIndices()

◆ getAltInstrMask()

◆ getBlendedLoadCost()

◆ getElementIndex()

◆ getExtractIndex()

◆ getGEPCosts()

◆ getInsertExtractIndex()

◆ getInsertExtractIndex< ExtractElementInst >()

◆ getInsertExtractIndex< InsertElementInst >()

◆ getLocation()

◆ getMaskedDivRemIntrinsic()

◆ getNumberOfPotentiallyCommutativeOps()

◆ getNumElements()

◆ getNumElems()

◆ getPartNumElems()

◆ getReassocCombineOpcode()

◆ getSameOpcode()

◆ getShuffleCost()

◆ getShufflevectorNumGroups()

◆ hasOnlyAbsorbableCopyableFMulOrFAdds()

◆ inversePermutation()

◆ isAbsorbableCopyableFMulOrFAdd()

◆ isAbsorbableFMulOrFAdd()

◆ isAlternateInstruction()

◆ isBinOpIdentityConstant()

◆ isCmpSameOrSwapped()

◆ isCommutableOperand()

◆ isCommutative() [1/2]

◆ isCommutative() [2/2]

◆ isConstant()

◆ isEquivalentIntrinsicID()

◆ isNonProfitableIndex()

◆ isOnceUsedSeed()

◆ isReassocChainLink()

◆ isSelectedBaseLoad()

◆ isSimple()

◆ isSplat()

◆ isUndefVector()

◆ isUndefVector< false >()

◆ isUndefVector< true >()

◆ isUsedOutsideBlock()

◆ isValidForAlternation()

◆ isVectorLikeInstWithConstOps()

◆ lookThroughCastRoundTrip()

◆ reorderScalars()

◆ replicateMask()

◆ scanAltAssociativeOperands()

◆ shortBundleName()

◆ transformScalarShuffleIndiciesToVector()

Variable Documentation

◆ UsesLimit