#include "Transforms/Vectorize/VPlanTransforms.h"

Static Public Member Functions
template<typename... ArgsTy>
static bool	runPass (bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
	Helper to run a VPlan transform `Transform` on `VPlan`, forwarding extra arguments to the transform.
template<typename... ArgsTy>
static void	runPass (void(*Fn)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
	Helper to run a VPlan transform `Transform` on `VPlan`, forwarding extra arguments to the transform.
static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan >	buildVPlan0 (Loop TheLoop, LoopInfo &LI, Type InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE, LoopVersioning *LVer=nullptr)
	Create a base VPlan0, serving as the common starting point for all later candidates.
static LLVM_ABI_FOR_TEST void	handleEarlyExits (VPlan &Plan, bool HasUncountableExit)
	Update `Plan` to account for all early exits.
static LLVM_ABI_FOR_TEST void	addMiddleCheck (VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)
	If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block.
static void	addMinimumIterationCheck (VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop OrigLoop, const uint32_t MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)
static void	addMinimumVectorEpilogueIterationCheck (VPlan &Plan, Value TripCount, Value VectorTripCount, bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE)
	Add a check to `Plan` to see if the epilogue vector loop should be executed.
static LLVM_ABI_FOR_TEST void	createLoopRegions (VPlan &Plan)
	Replace loops in `Plan's` flat CFG with VPRegionBlocks, turning `Plan's` flat CFG into a hierarchical CFG.
static void	attachCheckBlock (VPlan &Plan, Value Cond, BasicBlock CheckBlock, bool AddBranchWeights)
	Wrap runtime check block `CheckBlock` in a VPIRBB and `Cond` in a VPValue and connect the block to `Plan`, using the VPValue as branch condition.
static LLVM_ABI_FOR_TEST bool	tryToConvertVPInstructionsToVPRecipes (VPlan &Plan, function_ref< const InductionDescriptor (PHINode )> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)
	Replaces the VPInstructions in `Plan` with corresponding widen recipes.
static bool	handleMultiUseReductions (VPlan &Plan)
	Try to legalize reductions with multiple in-loop uses.
static bool	adjustFixedOrderRecurrences (VPlan &Plan, VPBuilder &Builder)
	Try to have all users of fixed-order recurrences appear after the recipe defining their previous value, by either sinking users or hoisting recipes defining their previous value (and its operands).
static bool	handleMaxMinNumReductions (VPlan &Plan)
	Check if `Plan` contains any FMaxNum or FMinNum reductions.
static void	clearReductionWrapFlags (VPlan &Plan)
	Clear NSW/NUW flags from reduction instructions if necessary.
static void	unrollByUF (VPlan &Plan, unsigned UF)
	Explicitly unroll `Plan` by `UF`.
static void	replicateByVF (VPlan &Plan, ElementCount VF)
	Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in `Plan` with `VF` single-scalar recipes.
static void	optimizeForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Optimize `Plan` based on `BestVF` and `BestUF`.
static LLVM_ABI_FOR_TEST void	optimize (VPlan &Plan)
	Apply VPlan-to-VPlan optimizations to `Plan`, including induction recipe optimizations, dead recipe removal, replicate region optimizations and block merging.
static void	createAndOptimizeReplicateRegions (VPlan &Plan)
	Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask operand.
static void	addActiveLaneMask (VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
	Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe, wide canonical IV, trip-count).
static void	truncateToMinimalBitwidths (VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
	Insert truncates and extends for any truncated recipe.
static void	replaceSymbolicStrides (VPlan &Plan, PredicatedScalarEvolution &PSE, const DenseMap< Value , const SCEV > &StridesMap)
	Replace symbolic strides from `StridesMap` in `Plan` with constants when possible.
static void	dropPoisonGeneratingRecipes (VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)
	Drop poison flags from recipes that may generate a poison value that is used after vectorization, even when their operands are not poison.
static void	addExplicitVectorLength (VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
	Add a VPEVLBasedIVPHIRecipe and related recipes to `Plan` and replaces all uses except the canonical IV increment of VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
static void	createInterleaveGroups (VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)
static void	removeDeadRecipes (VPlan &Plan)
	Remove dead recipes from `Plan`.
static void	handleUncountableEarlyExit (VPBasicBlock EarlyExitingVPBB, VPBasicBlock EarlyExitVPBB, VPlan &Plan, VPBasicBlock HeaderVPBB, VPBasicBlock LatchVPBB)
	Update `Plan` to account for the uncountable early exit from `EarlyExitingVPBB` to `EarlyExitVPBB` by.
static void	dissolveLoopRegions (VPlan &Plan)
	Replace loop regions with explicit CFG.
static void	canonicalizeEVLLoops (VPlan &Plan)
	Transform EVL loops to use variable-length stepping after region dissolution.
static void	convertToConcreteRecipes (VPlan &Plan)
	Lower abstract recipes to concrete ones, that can be codegen'd.
static void	convertToAbstractRecipes (VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
	This function converts initial recipes to the abstract recipes and clamps `Range` based on cost model for following optimizations and cost estimations.
static void	simplifyRecipes (VPlan &Plan)
	Perform instcombine-like simplifications on recipes in `Plan`.
static void	removeBranchOnConst (VPlan &Plan)
	Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their successors.
static void	cse (VPlan &Plan)
	Perform common-subexpression-elimination on `Plan`.
static void	optimizeInductionExitUsers (VPlan &Plan, DenseMap< VPValue , VPValue > &EndValues, ScalarEvolution &SE)
	If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them precomputed end values instead, possibly taken one step backwards.
static void	materializeBroadcasts (VPlan &Plan)
	Add explicit broadcasts for live-ins and VPValues defined in `Plan's` entry block if they are used as vectors.
static void	hoistInvariantLoads (VPlan &Plan)
	Hoist single-scalar loads with invariant addresses out of the vector loop to the preheader, if they are proven not to alias with any stores in the plan using noalias metadata.
static void	hoistPredicatedLoads (VPlan &Plan, ScalarEvolution &SE, const Loop *L)
	Hoist predicated loads from the same address to the loop entry block, if they are guaranteed to execute on both paths (i.e., in replicate regions with complementary masks P and NOT P).
static void	materializeConstantVectorTripCount (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
static void	materializeVectorTripCount (VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)
	Materialize vector trip count computations to a set of VPInstructions.
static void	materializeBackedgeTakenCount (VPlan &Plan, VPBasicBlock *VectorPH)
	Materialize the backedge-taken count to be computed explicitly using VPInstructions.
static void	materializePacksAndUnpacks (VPlan &Plan)
	Add explicit Build[Struct]Vector recipes to Pack multiple scalar values into vectors and Unpack recipes to extract scalars from vectors as needed.
static void	materializeVFAndVFxUF (VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
	Materialize VF and VFxUF to be computed explicitly using VPInstructions.
static DenseMap< const SCEV , Value >	expandSCEVs (VPlan &Plan, ScalarEvolution &SE)
	Expand VPExpandSCEVRecipes in `Plan's` entry block.
static void	narrowInterleaveGroups (VPlan &Plan, ElementCount VF, TypeSize VectorRegWidth)
	Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups replaced by wide loads and stores processing VF elements, if all transformed interleave groups access the full vector width (checked via \o VectorRegWidth).
static DenseMap< VPBasicBlock , VPValue >	introduceMasksAndLinearize (VPlan &Plan, bool FoldTail)
	Predicate and linearize the control-flow in the only loop region of `Plan`.
static void	addBranchWeightToMiddleTerminator (VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
	Add branch weight metadata, if the `Plan's` middle block is terminated by a BranchOnCond recipe.
static void	updateScalarResumePhis (VPlan &Plan, DenseMap< VPValue , VPValue > &IVEndValues)
	Update the resume phis in the scalar preheader after creating wide recipes for first-order recurrences, reductions and inductions.
static void	addExitUsersForFirstOrderRecurrences (VPlan &Plan, VFRange &Range)
	Handle users in the exit block for first order reductions in the original exit block.

Detailed Description

Definition at line 38 of file VPlanTransforms.h.

Member Function Documentation

◆ addActiveLaneMask()

void VPlanTransforms::addActiveLaneMask	(	VPlan &	Plan,
		bool	UseActiveLaneMaskForControlFlow,
		bool	DataAndControlFlowWithoutRuntimeCheck )

static

Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe, wide canonical IV, trip-count).

If UseActiveLaneMaskForControlFlow is true, introduce an VPActiveLaneMaskPHIRecipe. If DataAndControlFlowWithoutRuntimeCheck is true, no minimum-iteration runtime check will be created (during skeleton creation) and instead it is handled using active-lane-mask. DataAndControlFlowWithoutRuntimeCheck implies UseActiveLaneMaskForControlFlow.

Definition at line 2679 of file VPlanTransforms.cpp.

References llvm::VPInstruction::ActiveLaneMask, addVPLaneMaskPhiAndUpdateExitBranch(), assert(), B(), llvm::cast(), llvm::DataAndControlFlowWithoutRuntimeCheck, llvm::VPRecipeBase::eraseFromParent(), llvm::find_if(), findHeaderMask(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getOrAddLiveIn(), llvm::VPBuilder::getToInsertAfter(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::IsaPred, llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::users().

◆ addBranchWeightToMiddleTerminator()

void VPlanTransforms::addBranchWeightToMiddleTerminator	(	VPlan &	Plan,
		ElementCount	VF,
		std::optional< unsigned >	VScaleForTuning )

static

Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.

Definition at line 4774 of file VPlanTransforms.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::MDBuilder::createBranchWeights(), llvm::dyn_cast_or_null(), llvm::VPlan::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getMiddleBlock(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getUF(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ addExitUsersForFirstOrderRecurrences()

void VPlanTransforms::addExitUsersForFirstOrderRecurrences	(	VPlan &	Plan,
		VFRange &	Range )

static

Handle users in the exit block for first order reductions in the original exit block.

The penultimate value of recurrences is fed to their LCSSA phi users in the original exit block using the VPIRInstruction wrapping to the LCSSA phi.

Definition at line 4878 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::VPBuilder::createNaryOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractPenultimateElement, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getMiddleBlock(), llvm::ElementCount::getScalable(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_ExtractLastElement(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), Range, and llvm::VPValue::replaceAllUsesWith().

◆ addExplicitVectorLength()

void VPlanTransforms::addExplicitVectorLength	(	VPlan &	Plan,
		const std::optional< unsigned > &	MaxSafeElements )

static

Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV increment of VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.

VPCanonicalIVPHIRecipe is only used to control the loop after this transformation.

VPCanonicalIVPHIRecipe is used only for loop iterations counting after this transformation.

The function uses the following definitions: StartV is the canonical induction start value.

The function adds the following recipes:

vector.ph: ...

vector.body: ... EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ StartV, vector.ph ], [ NextEVLIV, vector.body ] AVL = phi [ trip-count, vector.ph ], [ NextAVL, vector.body ] VPEVL = EXPLICIT-VECTOR-LENGTH AVL ... OpEVL = cast i32 VPEVL to IVSize NextEVLIV = add IVSize OpEVL, EVLPhi NextAVL = sub IVSize nuw AVL, OpEVL ...

If MaxSafeElements is provided, the function adds the following recipes: vector.ph: ...

vector.body: ... EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ StartV, vector.ph ], [ NextEVLIV, vector.body ] AVL = phi [ trip-count, vector.ph ], [ NextAVL, vector.body ] cmp = cmp ult AVL, MaxSafeElements SAFE_AVL = select cmp, AVL, MaxSafeElements VPEVL = EXPLICIT-VECTOR-LENGTH SAFE_AVL ... OpEVL = cast i32 VPEVL to IVSize NextEVLIV = add IVSize OpEVL, EVLPhi NextAVL = sub IVSize nuw AVL, OpEVL ...

Definition at line 2992 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), llvm::cast(), llvm::VPInstruction::ExplicitVectorLength, llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getInt32Ty(), llvm::VPlan::getTripCount(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalarVFOnly(), llvm::CmpInst::ICMP_ULT, llvm::VPlan::setUF(), and transformRecipestoEVLRecipes().

◆ addMiddleCheck()

void VPlanTransforms::addMiddleCheck	(	VPlan &	Plan,
		bool	RequiresScalarEpilogueCheck,
		bool	TailFolded )

static

If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block.

Definition at line 615 of file VPlanConstruction.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::VPlan::getFalse(), llvm::VPBlockBase::getPredecessors(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::VPlan::getVectorTripCount(), and llvm::CmpInst::ICMP_EQ.

◆ addMinimumIterationCheck()

void VPlanTransforms::addMinimumIterationCheck	(	VPlan &	Plan,
		ElementCount	VF,
		unsigned	UF,
		ElementCount	MinProfitableTripCount,
		bool	RequiresScalarEpilogue,
		bool	TailFolded,
		bool	CheckNeededWithTailFolding,
		Loop *	OrigLoop,
		const uint32_t *	MinItersBypassWeights,
		DebugLoc	DL,
		ScalarEvolution &	SE )

static

Definition at line 711 of file VPlanConstruction.cpp.

References llvm::ScalarEvolution::applyLoopGuards(), llvm::ArrayRef(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::MDBuilder::createBranchWeights(), DL, llvm::SCEV::FlagNUW, llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::ScalarEvolution::getElementCount(), llvm::VPlan::getEntry(), llvm::VPlan::getFalse(), llvm::CmpInst::getInversePredicate(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getMask(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::ScalarEvolution::getUMaxExpr(), llvm::DebugLoc::getUnknown(), llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, llvm::ScalarEvolution::isKnownPredicate(), and MinItersBypassWeights.

Referenced by llvm::LoopVectorizationPlanner::addMinimumIterationCheck().

◆ addMinimumVectorEpilogueIterationCheck()

void VPlanTransforms::addMinimumVectorEpilogueIterationCheck	(	VPlan &	Plan,
		Value *	TripCount,
		Value *	VectorTripCount,
		bool	RequiresScalarEpilogue,
		ElementCount	EpilogueVF,
		unsigned	EpilogueUF,
		unsigned	MainLoopStep,
		unsigned	EpilogueLoopStep,
		ScalarEvolution &	SE )

static

Add a check to Plan to see if the epilogue vector loop should be executed.

Definition at line 795 of file VPlanConstruction.cpp.

References llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::Count, llvm::MDBuilder::createBranchWeights(), llvm::SCEV::FlagNUW, llvm::VPlan::getContext(), llvm::ScalarEvolution::getElementCount(), llvm::VPlan::getEntry(), llvm::VPlan::getOrAddLiveIn(), llvm::Value::getType(), llvm::DebugLoc::getUnknown(), llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, and P.

Referenced by preparePlanForEpilogueVectorLoop().

◆ adjustFixedOrderRecurrences()

bool VPlanTransforms::adjustFixedOrderRecurrences	(	VPlan &	Plan,
		VPBuilder &	Builder )

static

Try to have all users of fixed-order recurrences appear after the recipe defining their previous value, by either sinking users or hoisting recipes defining their previous value (and its operands).

Then introduce FirstOrderRecurrenceSplice VPInstructions to combine the value from the recurrence phis and previous values.

Returns: true if all users of fixed-order recurrences could be re-arranged as needed or false if it is not possible. In the latter case, Plan is not valid.

Definition at line 2146 of file VPlanTransforms.cpp.

References assert(), B(), llvm::cast(), llvm::VPBuilder::createNaryOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPInstruction::ExtractLane, llvm::VPInstruction::ExtractLastElement, llvm::VPInstruction::FirstOrderRecurrenceSplice, llvm::VPlan::getContext(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::Type::getInt64Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPlan::getOrAddLiveIn(), llvm::VPRecipeBase::getParent(), llvm::VPlan::getVectorLoopRegion(), hoistPreviousBeforeFORUsers(), llvm::CmpInst::ICMP_EQ, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPBuilder::setInsertPoint(), and sinkRecurrenceUsersAfterPrevious().

◆ attachCheckBlock()

void VPlanTransforms::attachCheckBlock	(	VPlan &	Plan,
		Value *	Cond,
		BasicBlock *	CheckBlock,
		bool	AddBranchWeights )

static

Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan, using the VPValue as branch condition.

Definition at line 675 of file VPlanConstruction.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::cast(), CheckBypassWeights, Cond, llvm::VPBlockUtils::connectBlocks(), llvm::MDBuilder::createBranchWeights(), llvm::VPBuilder::createNaryOp(), llvm::VPlan::createVPIRBasicBlock(), llvm::VPlan::getContext(), getDebugLoc(), llvm::VPBlockBase::getNumPredecessors(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPBlockUtils::insertOnEdge(), llvm::isa(), and llvm::VPBlockBase::swapSuccessors().

◆ buildVPlan0()

std::unique_ptr< VPlan > VPlanTransforms::buildVPlan0	(	Loop *	TheLoop,
		LoopInfo &	LI,
		Type *	InductionTy,
		DebugLoc	IVDL,
		PredicatedScalarEvolution &	PSE,
		LoopVersioning *	LVer = nullptr )

static

Create a base VPlan0, serving as the common starting point for all later candidates.

It consists of an initial plain CFG loop with loop blocks from TheLoop being directly translated to VPBasicBlocks with VPInstruction corresponding to the input IR.

The created loop is wrapped in an initial skeleton to facilitate vectorization, consisting of a vector pre-header, an exit block for the main vector loop (middle.block) and a new block as preheader of the scalar loop (scalar.ph). See below for an illustration. It also adds a canonical IV and its increment, using InductionTy and IVDL, and creates a VPValue expression for the original trip count.

[ ] <– Plan's entry VPIRBasicBlock, wrapping the original loop's / \ old preheader. Will contain iteration number check and SCEV | | expansions. | | / v | [ ] <– vector loop bypass (may consist of multiple blocks) will be | / | added later. | / v || [ ] <– vector pre header. |/ | | v | [ ] \ <– plain CFG loop wrapping original loop to be vectorized. | [ ]_| | | | v | [ ] <— middle-block with the branch to successors | / | | / | | | v --->[ ] <— scalar preheader (initial a VPBasicBlock, which will be | | replaced later by a VPIRBasicBlock wrapping the scalar | | preheader basic block. | | v <– edge from middle to exit iff epilogue is not required. | [ ] \ | [ ]_| <– old scalar loop to handle remainder (scalar epilogue, | | header wrapped in VPIRBasicBlock). \ | \ v >[ ] <– original loop exit block(s), wrapped in VPIRBasicBlocks.

Definition at line 570 of file VPlanConstruction.cpp.

References addInitialSkeleton().

◆ canonicalizeEVLLoops()

void VPlanTransforms::canonicalizeEVLLoops ( VPlan & Plan )

static

Transform EVL loops to use variable-length stepping after region dissolution.

Once loop regions are replaced with explicit CFG, EVL loops can step with variable vector lengths instead of fixed lengths. This transformation:

Makes EVL-Phi concrete.

Replaces the exit condition from (branch-on-count CanonicalIVInc, VectorTripCount) to (branch-on-cond eq AVLNext, 0)

Definition at line 3054 of file VPlanTransforms.cpp.

References assert(), llvm::VPBasicBlock::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::VPBuilder::createScalarPhi(), llvm::dyn_cast(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPHeaderPHIRecipe::getBackedgeValue(), llvm::VPlan::getConstantInt(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPlan::getEntry(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getPredecessors(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_EVL(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSelf(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ clearReductionWrapFlags()

void VPlanTransforms::clearReductionWrapFlags ( VPlan & Plan )

static

Clear NSW/NUW flags from reduction instructions if necessary.

Definition at line 2219 of file VPlanTransforms.cpp.

References llvm::Add, llvm::AddChainWithSubs, collectUsersRecursively(), llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::Mul, llvm::VPBasicBlock::phis(), and llvm::Sub.

◆ convertToAbstractRecipes()

void VPlanTransforms::convertToAbstractRecipes	(	VPlan &	Plan,
		VPCostContext &	Ctx,
		VFRange &	Range )

static

This function converts initial recipes to the abstract recipes and clamps Range based on cost model for following optimizations and cost estimations.

The converted abstract recipes will lower to concrete recipes before codegen.

Definition at line 4026 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPlan::getVectorLoopRegion(), llvm::make_early_inc_range(), Range, tryToCreateAbstractReductionRecipe(), and llvm::vp_depth_first_deep().

◆ convertToConcreteRecipes()

void VPlanTransforms::convertToConcreteRecipes ( VPlan & Plan )

static

Lower abstract recipes to concrete ones, that can be codegen'd.

Definition at line 3577 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::dyn_cast(), expandVPWidenIntOrFpInduction(), expandVPWidenPointerInduction(), llvm::VPInstruction::FirstActiveLane, llvm::VPlan::getContext(), llvm::VPlan::getEntry(), llvm::Type::getInt64Ty(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::hasScalableVF(), I, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), llvm::VPInstruction::LastActiveLane, llvm::PatternMatch::m_One(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::Mul, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceAllUsesWith(), scalarizeVPWidenPointerInduction(), Select, ToRemove, and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ createAndOptimizeReplicateRegions()

void VPlanTransforms::createAndOptimizeReplicateRegions ( VPlan & Plan )

static

Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask operand.

Optimize the created regions by iteratively sinking scalar operands into the region, followed by merging regions until no improvements are remaining.

Definition at line 518 of file VPlanTransforms.cpp.

References addReplicateRegions(), mergeBlocksIntoPredecessors(), mergeReplicateRegionsIntoSuccessors(), and sinkScalarOperands().

Referenced by optimize().

◆ createInterleaveGroups()

void VPlanTransforms::createInterleaveGroups	(	VPlan &	Plan,
		const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &	InterleaveGroups,
		VPRecipeBuilder &	RecipeBuilder,
		const bool &	ScalarEpilogueAllowed )

static

◆ createLoopRegions()

void VPlanTransforms::createLoopRegions ( VPlan & Plan )

static

Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG.

Definition at line 660 of file VPlanConstruction.cpp.

References canonicalHeaderAndLatch(), createLoopRegion(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::VPBlockBase::setName(), and llvm::vp_post_order_shallow().

◆ cse()

void VPlanTransforms::cse ( VPlan & Plan )

static

Perform common-subexpression-elimination on Plan.

Perform a common-subexpression-elimination of VPSingleDefRecipes on the Plan.

Definition at line 2328 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ dissolveLoopRegions()

void VPlanTransforms::dissolveLoopRegions ( VPlan & Plan )

static

Replace loop regions with explicit CFG.

Definition at line 3565 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::VPlan::getEntry(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ dropPoisonGeneratingRecipes()

void VPlanTransforms::dropPoisonGeneratingRecipes	(	VPlan &	Plan,
		const std::function< bool(BasicBlock *)> &	BlockNeedsPredication )

static

Drop poison flags from recipes that may generate a poison value that is used after vectorization, even when their operands are not poison.

Those recipes meet the following conditions:

Contribute to the address computation of a recipe generating a widen memory load/store (VPWidenMemoryInstructionRecipe or VPInterleaveRecipe).
Such a widen memory load/store has at least one underlying Instruction that is in a basic block that needs predication and after vectorization the generated instruction won't be predicated. Uses BlockNeedsPredication to check if a block needs predicating. TODO: Replace BlockNeedsPredication callback with retrieving info from VPlan directly.

Definition at line 3190 of file VPlanTransforms.cpp.

References A(), assert(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPlan::getEntry(), llvm::InterleaveGroup< InstTy >::getMember(), llvm::InterleaveGroup< InstTy >::getNumMembers(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::VPValue::getUnderlyingValue(), llvm::VPDef::getVPSingleValue(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPUser::operands(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::vp_depth_first_deep().

◆ expandSCEVs()

DenseMap< const SCEV , Value > VPlanTransforms::expandSCEVs	(	VPlan &	Plan,
		ScalarEvolution &	SE )

static

Expand VPExpandSCEVRecipes in Plan's entry block.

Each VPExpandSCEVRecipe is replaced with a live-in wrapping the expanded IR value. A mapping from SCEV expressions to their expanded IR value is returned.

Definition at line 4478 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::VPIRInstruction::create(), DL, llvm::drop_end(), llvm::dyn_cast(), llvm::SCEVExpander::expandCodeFor(), llvm::ScalarEvolution::getDataLayout(), llvm::VPlan::getEntry(), llvm::VPlan::getOrAddLiveIn(), llvm::BasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), I, llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::IsaPred, llvm::make_early_inc_range(), llvm::none_of(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPlan::resetTripCount().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ handleEarlyExits()

void VPlanTransforms::handleEarlyExits	(	VPlan &	Plan,
		bool	HasUncountableExit )

static

Update Plan to account for all early exits.

Definition at line 579 of file VPlanConstruction.cpp.

References assert(), llvm::cast(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPlan::getExitBlocks(), llvm::VPBlockBase::getPredecessors(), llvm::VPlan::getScalarHeader(), llvm::VPBlockBase::getSinglePredecessor(), handleUncountableEarlyExit(), and llvm::to_vector().

◆ handleMaxMinNumReductions()

bool VPlanTransforms::handleMaxMinNumReductions ( VPlan & Plan )

static

Check if Plan contains any FMaxNum or FMinNum reductions.

If they do, try to update the vector loop to exit early if any input is NaN and resume executing in the scalar loop to handle the NaNs there. Return false if this attempt was unsuccessful.

Check if the vector loop of Plan can early exit and restart execution of last vector iteration in the scalar loop. This requires all recipes up to early exit point be side-effect free as they are re-executed. Currently we check that the loop is free of any recipe that may write to memory. Expected to operate on an early VPlan w/o nested regions.

Definition at line 845 of file VPlanConstruction.cpp.

References _, llvm::VPInstruction::AnyOf, assert(), llvm::VPBasicBlock::begin(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBuilder::createAnd(), llvm::VPBuilder::createFCmp(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createNot(), llvm::VPBuilder::createOr(), llvm::VPBuilder::createSelect(), llvm::dbgs(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPRecipeBase::eraseFromParent(), llvm::CmpInst::FCMP_UNO, findUserOf(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBuilder::getInsertPoint(), llvm::VPlan::getMiddleBlock(), llvm::getMinMaxReductionIntrinsicOp(), llvm::VPlan::getScalarPreheader(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::hasScalarTail(), llvm::CmpInst::ICMP_EQ, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(), LLVM_DEBUG, llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::PatternMatch::m_Intrinsic(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPRecipeBase::moveAfter(), llvm::VPBasicBlock::phis(), llvm::VPBuilder::setInsertPoint(), llvm::VPUser::setOperand(), and llvm::vp_depth_first_shallow().

◆ handleMultiUseReductions()

bool VPlanTransforms::handleMultiUseReductions ( VPlan & Plan )

static

Try to legalize reductions with multiple in-loop uses.

Currently only min/max reductions used by FindLastIV reductions are supported. Otherwise return false.

Definition at line 1000 of file VPlanConstruction.cpp.

References assert(), B(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), findUserOf(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::CmpInst::getInversePredicate(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::getMinMaxReductionIntrinsicOp(), llvm::VPValue::getNumUsers(), llvm::VPUser::getOperand(), llvm::VPRecipeBase::getParent(), llvm::CmpInst::getSwappedPredicate(), llvm::VPlan::getVectorLoopRegion(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_ULE, llvm::is_contained(), llvm::isa(), llvm::RecurrenceDescriptor::isFindLastIVRecurrenceKind(), llvm::RecurrenceDescriptor::isIntMinMaxRecurrenceKind(), llvm_unreachable, llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_TruncOrSelf(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPRecipeBase::moveBefore(), llvm::VPBasicBlock::phis(), llvm::Sentinel, llvm::VPUser::setOperand(), llvm::SMax, llvm::SMin, std::swap(), llvm::UMax, and llvm::UMin.

◆ handleUncountableEarlyExit()

void VPlanTransforms::handleUncountableEarlyExit	(	VPBasicBlock *	EarlyExitingVPBB,
		VPBasicBlock *	EarlyExitVPBB,
		VPlan &	Plan,
		VPBasicBlock *	HeaderVPBB,
		VPBasicBlock *	LatchVPBB )

static

Update Plan to account for the uncountable early exit from EarlyExitingVPBB to EarlyExitVPBB by.

updating the condition exiting the loop via the latch to include the early exit condition,
splitting the original middle block to branch to the early exit block conditionally - according to the early exit condition.

Definition at line 3690 of file VPlanTransforms.cpp.

References llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::BranchOnCount, llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::VPBuilder::createNaryOp(), llvm::VPlan::createVPBasicBlock(), llvm::VPInstruction::ExtractLane, llvm::VPInstruction::FirstActiveLane, llvm::VPBlockBase::getNumPredecessors(), llvm::VPUser::getOperand(), llvm::VPBlockBase::getPredecessors(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPBasicBlock::getTerminator(), llvm::CmpInst::ICMP_EQ, llvm::VPBlockUtils::insertOnEdge(), llvm::VPValue::isLiveIn(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::VPUser::setOperand(), and llvm::VPBlockBase::swapSuccessors().

Referenced by handleEarlyExits().

◆ hoistInvariantLoads()

void VPlanTransforms::hoistInvariantLoads ( VPlan & Plan )

static

Hoist single-scalar loads with invariant addresses out of the vector loop to the preheader, if they are proven not to alias with any stores in the plan using noalias metadata.

Definition at line 4082 of file VPlanTransforms.cpp.

References llvm::MemoryLocation::AATags, llvm::all_of(), llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::vputils::getMemoryLocation(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPValue::isDefinedOutsideLoopRegions(), llvm::ScopedNoAliasAAResult::mayAliasInScopes(), llvm::AAMDNodes::NoAlias, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AAMDNodes::Scope, and llvm::vp_depth_first_shallow().

Referenced by optimize().

◆ hoistPredicatedLoads()

void VPlanTransforms::hoistPredicatedLoads	(	VPlan &	Plan,
		ScalarEvolution &	SE,
		const Loop *	L )

static

Hoist predicated loads from the same address to the loop entry block, if they are guaranteed to execute on both paths (i.e., in replicate regions with complementary masks P and NOT P).

Definition at line 4138 of file VPlanTransforms.cpp.

References A(), llvm::any_of(), B(), llvm::VPBasicBlock::back(), llvm::Block, canHoistLoadWithNoAliasCheck(), llvm::cast(), llvm::drop_begin(), llvm::dyn_cast(), getCommonLoadMetadata(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getParent(), llvm::VPRecipeBase::getParent(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPTypeAnalysis::inferScalarType(), llvm::isa(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::min_element(), llvm::VPDominatorTree::properlyDominates(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::sort(), and llvm::vp_depth_first_shallow().

◆ introduceMasksAndLinearize()

DenseMap< VPBasicBlock , VPValue > VPlanTransforms::introduceMasksAndLinearize	(	VPlan &	Plan,
		bool	FoldTail )

static

Predicate and linearize the control-flow in the only loop region of Plan.

If FoldTail is true, create a mask guarding the loop header, otherwise use all-true for the header mask. Masks for blocks are added to a block-to-mask map which is returned in order to be used later for wide recipe construction. This argument is temporary and will be removed in the future.

Definition at line 265 of file VPlanPredicator.cpp.

References assert(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPInstruction::ExtractLane, llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getExitBlocks(), llvm::VPlan::getMiddleBlock(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPInstruction::LastActiveLane, llvm::VPlanPatternMatch::m_ExtractLastElement(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), P, llvm::VPBasicBlock::phis(), Predicator, llvm::VPValue::replaceAllUsesWith(), and llvm::to_vector().

◆ materializeBackedgeTakenCount()

void VPlanTransforms::materializeBackedgeTakenCount	(	VPlan &	Plan,
		VPBasicBlock *	VectorPH )

static

Materialize the backedge-taken count to be computed explicitly using VPInstructions.

Definition at line 4276 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::begin(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPlan::getConstantInt(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPlan::getTripCount(), llvm::VPTypeAnalysis::inferScalarType(), and llvm::VPValue::replaceAllUsesWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeBroadcasts()

void VPlanTransforms::materializeBroadcasts ( VPlan & Plan )

static

Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as vectors.

Definition at line 4037 of file VPlanTransforms.cpp.

References llvm::append_range(), assert(), llvm::VPBasicBlock::begin(), llvm::VPInstruction::Broadcast, llvm::cast(), llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::VPlan::getEntry(), llvm::VPlan::getLiveIns(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrCreateBackedgeTakenCount(), getParent(), llvm::VPlan::getVectorPreheader(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::vputils::onlyScalarValuesUsed(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Value::users().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeConstantVectorTripCount()

void VPlanTransforms::materializeConstantVectorTripCount	(	VPlan &	Plan,
		ElementCount	BestVF,
		unsigned	BestUF,
		PredicatedScalarEvolution &	PSE )

static

◆ materializePacksAndUnpacks()

void VPlanTransforms::materializePacksAndUnpacks ( VPlan & Plan )

static

Add explicit Build[Struct]Vector recipes to Pack multiple scalar values into vectors and Unpack recipes to extract scalars from vectors as needed.

Definition at line 4290 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::VPInstruction::BuildStructVector, llvm::VPInstruction::BuildVector, llvm::cast(), llvm::concat(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalarVFOnly(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertAfter(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::VPRegionBlock::isReplicator(), llvm::vputils::isSingleScalar(), llvm::Type::isStructTy(), llvm::make_early_inc_range(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPInstruction::Unpack, and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeVectorTripCount()

void VPlanTransforms::materializeVectorTripCount	(	VPlan &	Plan,
		VPBasicBlock *	VectorPHVPBB,
		bool	TailByMasking,
		bool	RequiresScalarEpilogue )

static

Materialize vector trip count computations to a set of VPInstructions.

Definition at line 4383 of file VPlanTransforms.cpp.

References assert(), llvm::VPBasicBlock::begin(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPlan::getConstantInt(), llvm::VPValue::getLiveInIRValue(), llvm::VPValue::getNumUsers(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPValue::isLiveIn(), and llvm::VPValue::replaceAllUsesWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeVFAndVFxUF()

void VPlanTransforms::materializeVFAndVFxUF	(	VPlan &	Plan,
		VPBasicBlock *	VectorPH,
		ElementCount	VF )

static

Materialize VF and VFxUF to be computed explicitly using VPInstructions.

Definition at line 4443 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::begin(), llvm::VPInstruction::Broadcast, llvm::VPlan::getConstantInt(), llvm::VPValue::getNumUsers(), llvm::VPlan::getTripCount(), llvm::VPlan::getUF(), llvm::VPlan::getVF(), llvm::VPlan::getVFxUF(), llvm::VPTypeAnalysis::inferScalarType(), llvm::vputils::onlyScalarValuesUsed(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::replaceUsesWithIf().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ narrowInterleaveGroups()

void VPlanTransforms::narrowInterleaveGroups	(	VPlan &	Plan,
		ElementCount	VF,
		TypeSize	VectorRegWidth )

static

Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups replaced by wide loads and stores processing VF elements, if all transformed interleave groups access the full vector width (checked via \o VectorRegWidth).

This effectively is a very simple form of loop-aware SLP, where we use interleave groups to identify candidates.

Definition at line 4637 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::any_of(), llvm::cast(), llvm::VPBuilder::createElementCount(), llvm::VPBuilder::createNaryOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::enumerate(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getNumSuccessors(), llvm::VPlan::getOrAddLiveIn(), llvm::ElementCount::getScalable(), llvm::VPlan::getUF(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPlan::getVF(), I, llvm::isa(), isAlreadyNarrow(), isConsecutiveInterleaveGroup(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), narrowInterleaveGroupOp(), llvm::vputils::onlyFirstLaneUsed(), P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), removeDeadRecipes(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPUser::setOperand().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ optimize()

void VPlanTransforms::optimize ( VPlan & Plan )

static

Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations, dead recipe removal, replicate region optimizations and block merging.

Definition at line 2506 of file VPlanTransforms.cpp.

References createAndOptimizeReplicateRegions(), hoistInvariantLoads(), legalizeAndOptimizeInductions(), licm, mergeBlocksIntoPredecessors(), narrowToSingleScalarRecipes(), removeBranchOnConst(), removeDeadRecipes(), removeRedundantCanonicalIVs(), removeRedundantExpandSCEVRecipes(), removeRedundantInductionCasts(), runPass(), simplifyBlends(), and simplifyRecipes().

Referenced by llvm::LoopVectorizationPlanner::buildVPlans().

◆ optimizeForVFAndUF()

void VPlanTransforms::optimizeForVFAndUF	(	VPlan &	Plan,
		ElementCount	BestVF,
		unsigned	BestUF,
		PredicatedScalarEvolution &	PSE )

static

Optimize Plan based on BestVF and BestUF.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 1979 of file VPlanTransforms.cpp.

References assert(), llvm::VPlan::getUF(), llvm::VPlan::hasUF(), llvm::VPlan::hasVF(), optimizeVectorInductionWidthForTCAndVFUF(), llvm::VPlan::setVF(), simplifyBranchConditionForVFAndUF(), simplifyKnownEVL(), and tryToReplaceALMWithWideALM().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ optimizeInductionExitUsers()

void VPlanTransforms::optimizeInductionExitUsers	(	VPlan &	Plan,
		DenseMap< VPValue , VPValue > &	EndValues,
		ScalarEvolution &	SE )

static

If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them precomputed end values instead, possibly taken one step backwards.

Definition at line 966 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::enumerate(), llvm::VPlan::getExitBlocks(), llvm::VPlan::getMiddleBlock(), optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().

◆ removeBranchOnConst()

void VPlanTransforms::removeBranchOnConst ( VPlan & Plan )

static

Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their successors.

Definition at line 2472 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), Cond, llvm::count(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getPredecessors(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_False(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), and optimize().

◆ removeDeadRecipes()

void VPlanTransforms::removeDeadRecipes ( VPlan & Plan )

static

Remove dead recipes from Plan.

Definition at line 625 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPlan::getEntry(), isDeadRecipe(), llvm::make_early_inc_range(), llvm::reverse(), and llvm::vp_post_order_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), narrowInterleaveGroups(), optimize(), preparePlanForMainVectorLoop(), and unrollByUF().

◆ replaceSymbolicStrides()

void VPlanTransforms::replaceSymbolicStrides	(	VPlan &	Plan,
		PredicatedScalarEvolution &	PSE,
		const DenseMap< Value , const SCEV > &	StridesMap )

static

Replace symbolic strides from StridesMap in Plan with constants when possible.

Definition at line 3134 of file VPlanTransforms.cpp.

References llvm::CallingConv::C, llvm::cast(), llvm::dyn_cast(), llvm::VPlan::getConstantInt(), llvm::VPlan::getEntry(), llvm::VPlan::getLiveIn(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::PredicatedScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::isa(), llvm::PatternMatch::match(), llvm::VPValue::replaceUsesWithIf(), llvm::VPlan::resetTripCount(), llvm::SCEVParameterRewriter::rewrite(), llvm::APInt::sext(), llvm::Value::users(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::values(), and llvm::APInt::zext().

◆ replicateByVF()

void VPlanTransforms::replicateByVF	(	VPlan &	Plan,
		ElementCount	VF )

static

Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in Plan with VF single-scalar recipes.

TODO: Also replicate VPScalarIVSteps and VPReplicateRecipes inside replicate regions, thereby dissolving the latter.

Create single-scalar version of DefR for all lanes.

Users that only demand the first lane can use the definition for lane

Definition at line 540 of file VPlanUnroll.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::VPInstruction::BuildStructVector, llvm::VPInstruction::BuildVector, llvm::cast(), cloneForLane(), llvm::concat(), llvm::drop_begin(), llvm::dyn_cast(), llvm::IntegerType::get(), llvm::BasicBlock::getContext(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getEntry(), llvm::VPIRBasicBlock::getIRBasicBlock(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getOpcode(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getVectorLoopRegion(), I, llvm::isa(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reverse(), llvm::to_vector(), ToRemove, llvm::VPInstruction::Unpack, and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ runPass() [1/2]

template<typename... ArgsTy>

bool llvm::VPlanTransforms::runPass	(	bool(*	Transform )(VPlan &, ArgsTy...),
		VPlan &	Plan,
		typename std::remove_reference< ArgsTy >::type &...	Args )

inlinestatic

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Returns the boolean returned by the transform.

Definition at line 42 of file VPlanTransforms.h.

References llvm::VerifyEachVPlan, and llvm::verifyVPlanIsValid().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), optimize(), preparePlanForMainVectorLoop(), and llvm::LoopVectorizePass::processLoop().

◆ runPass() [2/2]

template<typename... ArgsTy>

void llvm::VPlanTransforms::runPass	(	void(*	Fn )(VPlan &, ArgsTy...),
		VPlan &	Plan,
		typename std::remove_reference< ArgsTy >::type &...	Args )

inlinestatic

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Definition at line 52 of file VPlanTransforms.h.

References llvm::VerifyEachVPlan, and llvm::verifyVPlanIsValid().

◆ simplifyRecipes()

void VPlanTransforms::simplifyRecipes ( VPlan & Plan )

static

Perform instcombine-like simplifications on recipes in Plan.

Definition at line 1477 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::make_early_inc_range(), and simplifyRecipe().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), optimize(), and simplifyBranchConditionForVFAndUF().

◆ truncateToMinimalBitwidths()

void VPlanTransforms::truncateToMinimalBitwidths	(	VPlan &	Plan,
		const MapVector< Instruction *, uint64_t > &	MinBWs )

static

Insert truncates and extends for any truncated recipe.

Redundant casts will be folded later.

Definition at line 2379 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast_or_null(), llvm::dyn_cast(), llvm::IntegerType::get(), llvm::VPlan::getContext(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getUnderlyingValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPlan::hasScalarVFOnly(), llvm::VPTypeAnalysis::inferScalarType(), llvm::isa(), llvm::Type::isIntegerTy(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::lookup(), llvm::PatternMatch::m_ICmp(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), llvm::VPBuilder::setInsertPoint(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace(), and llvm::vp_depth_first_deep().

◆ tryToConvertVPInstructionsToVPRecipes()

bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes	(	VPlan &	Plan,
		function_ref< const InductionDescriptor (PHINode )>	GetIntOrFpInductionDescriptor,
		const TargetLibraryInfo &	TLI )

static

Replaces the VPInstructions in Plan with corresponding widen recipes.

Returns false if any VPInstructions could not be converted to a wide recipe if needed.

Definition at line 48 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::drop_end(), llvm::dyn_cast(), GEP, llvm::vputils::getFlagsFromIndDesc(), llvm::VPDef::getNumDefinedValues(), llvm::VPlan::getOrAddLiveIn(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::VPValue::getUnderlyingValue(), llvm::getVectorIntrinsicIDForCall(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), llvm::VPDef::getVPSingleValue(), II, llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::make_early_inc_range(), llvm::make_range(), llvm::Intrinsic::not_intrinsic, llvm::User::operands(), and llvm::VPValue::replaceAllUsesWith().

◆ unrollByUF()

void VPlanTransforms::unrollByUF	(	VPlan &	Plan,
		unsigned	UF )

static

Explicitly unroll Plan by UF.

Definition at line 412 of file VPlanUnroll.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::VPInstruction::CanonicalIVIncrementForPart, Cleanup, llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), H, llvm::isa(), llvm::make_early_inc_range(), llvm::make_scope_exit(), llvm::VPBasicBlock::phis(), removeDeadRecipes(), llvm::VPlan::setUF(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ updateScalarResumePhis()

void VPlanTransforms::updateScalarResumePhis	(	VPlan &	Plan,
		DenseMap< VPValue , VPValue > &	IVEndValues )

static

Update the resume phis in the scalar preheader after creating wide recipes for first-order recurrences, reductions and inductions.

End values for inductions are added to IVEndValues.

Definition at line 4831 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::VPBuilder::createNaryOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastElement, llvm::VPlan::getMiddleBlock(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::isa(), llvm::VPBasicBlock::phis(), llvm::VPInstruction::setName(), and tryToComputeEndValueForInduction().

The documentation for this struct was generated from the following files:

lib/Transforms/Vectorize/VPlanTransforms.h
lib/Transforms/Vectorize/VPlanConstruction.cpp
lib/Transforms/Vectorize/VPlanPredicator.cpp
lib/Transforms/Vectorize/VPlanTransforms.cpp
lib/Transforms/Vectorize/VPlanUnroll.cpp

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ addActiveLaneMask()

◆ addBranchWeightToMiddleTerminator()

◆ addExitUsersForFirstOrderRecurrences()

◆ addExplicitVectorLength()

◆ addMiddleCheck()

◆ addMinimumIterationCheck()

◆ addMinimumVectorEpilogueIterationCheck()

◆ adjustFixedOrderRecurrences()

◆ attachCheckBlock()

◆ buildVPlan0()

◆ canonicalizeEVLLoops()

◆ clearReductionWrapFlags()

◆ convertToAbstractRecipes()

◆ convertToConcreteRecipes()

◆ createAndOptimizeReplicateRegions()

◆ createInterleaveGroups()

◆ createLoopRegions()

◆ cse()

◆ dissolveLoopRegions()

◆ dropPoisonGeneratingRecipes()

◆ expandSCEVs()

◆ handleEarlyExits()

◆ handleMaxMinNumReductions()

◆ handleMultiUseReductions()

◆ handleUncountableEarlyExit()

◆ hoistInvariantLoads()

◆ hoistPredicatedLoads()

◆ introduceMasksAndLinearize()

◆ materializeBackedgeTakenCount()

◆ materializeBroadcasts()

◆ materializeConstantVectorTripCount()

◆ materializePacksAndUnpacks()

◆ materializeVectorTripCount()

◆ materializeVFAndVFxUF()

◆ narrowInterleaveGroups()

◆ optimize()

◆ optimizeForVFAndUF()

◆ optimizeInductionExitUsers()

◆ removeBranchOnConst()

◆ removeDeadRecipes()

◆ replaceSymbolicStrides()

◆ replicateByVF()

◆ runPass() [1/2]

◆ runPass() [2/2]

◆ simplifyRecipes()

◆ truncateToMinimalBitwidths()

◆ tryToConvertVPInstructionsToVPRecipes()

◆ unrollByUF()

◆ updateScalarResumePhis()