LLVM 22.0.0git
|
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient output IR, including which branches, basic-blocks and output IR instructions to generate, and their cost. More...
#include "Transforms/Vectorize/VPlan.h"
Public Member Functions | |
VPlan (Loop *L) | |
Construct a VPlan for L . | |
VPlan (BasicBlock *ScalarHeaderBB, VPValue *TC) | |
Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock wrapping ScalarHeaderBB and a trip count of TC . | |
LLVM_ABI_FOR_TEST | ~VPlan () |
void | setEntry (VPBasicBlock *VPBB) |
void | execute (VPTransformState *State) |
Generate the IR code for this VPlan. | |
InstructionCost | cost (ElementCount VF, VPCostContext &Ctx) |
Return the cost of this plan. | |
VPBasicBlock * | getEntry () |
const VPBasicBlock * | getEntry () const |
VPBasicBlock * | getVectorPreheader () |
Returns the preheader of the vector loop region, if one exists, or null otherwise. | |
LLVM_ABI_FOR_TEST VPRegionBlock * | getVectorLoopRegion () |
Returns the VPRegionBlock of the vector loop. | |
LLVM_ABI_FOR_TEST const VPRegionBlock * | getVectorLoopRegion () const |
VPBasicBlock * | getMiddleBlock () |
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar tail loop or the exit block from the loop latch. | |
const VPBasicBlock * | getMiddleBlock () const |
VPBasicBlock * | getScalarPreheader () const |
Return the VPBasicBlock for the preheader of the scalar loop. | |
VPIRBasicBlock * | getScalarHeader () const |
Return the VPIRBasicBlock wrapping the header of the scalar loop. | |
ArrayRef< VPIRBasicBlock * > | getExitBlocks () const |
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop. | |
VPIRBasicBlock * | getExitBlock (BasicBlock *IRBB) const |
Return the VPIRBasicBlock corresponding to IRBB . | |
bool | isExitBlock (VPBlockBase *VPBB) |
Returns true if VPBB is an exit block. | |
VPValue * | getTripCount () const |
The trip count of the original loop. | |
void | setTripCount (VPValue *NewTripCount) |
Set the trip count assuming it is currently null; if it is not - use resetTripCount(). | |
void | resetTripCount (VPValue *NewTripCount) |
Resets the trip count for the VPlan. | |
VPValue * | getOrCreateBackedgeTakenCount () |
The backedge taken count of the original loop. | |
VPValue * | getBackedgeTakenCount () const |
VPValue & | getVectorTripCount () |
The vector trip count. | |
VPValue & | getVF () |
Returns the VF of the vector loop region. | |
const VPValue & | getVF () const |
VPValue & | getSymbolicUF () |
Returns the symbolic UF of the vector loop region. | |
VPValue & | getVFxUF () |
Returns VF * UF of the vector loop region. | |
LLVMContext & | getContext () const |
void | addVF (ElementCount VF) |
void | removeVF (ElementCount VF) |
Remove VF from the plan. | |
void | setVF (ElementCount VF) |
bool | hasVF (ElementCount VF) const |
bool | hasScalableVF () const |
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > | vectorFactors () const |
Returns an iterator range over all VFs of the plan. | |
bool | hasScalarVFOnly () const |
bool | hasUF (unsigned UF) const |
unsigned | getUF () const |
void | setUF (unsigned UF) |
bool | isUnrolled () const |
Returns true if the VPlan already has been unrolled, i.e. | |
std::string | getName () const |
Return a string with the name of the plan and the applicable VFs and UFs. | |
void | setName (const Twine &newName) |
VPValue * | getOrAddLiveIn (Value *V) |
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V . | |
VPValue * | getTrue () |
Return a VPValue wrapping i1 true. | |
VPValue * | getFalse () |
Return a VPValue wrapping i1 false. | |
VPValue * | getLiveIn (Value *V) const |
Return the live-in VPValue for V , if there is one or nullptr otherwise. | |
ArrayRef< VPValue * > | getLiveIns () const |
Return the list of live-in VPValues available in the VPlan. | |
void | printLiveIns (raw_ostream &O) const |
Print the live-ins of this VPlan to O . | |
void | print (raw_ostream &O) const |
Print this VPlan to O . | |
void | printDOT (raw_ostream &O) const |
Print this VPlan in DOT format to O . | |
LLVM_DUMP_METHOD void | dump () const |
Dump the plan to stderr (for debugging). | |
VPValue * | getSCEVExpansion (const SCEV *S) const |
void | addSCEVExpansion (const SCEV *S, VPValue *V) |
VPlan * | duplicate () |
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clones, and return it. | |
VPBasicBlock * | createVPBasicBlock (const Twine &Name, VPRecipeBase *Recipe=nullptr) |
Create a new VPBasicBlock with Name and containing Recipe if present. | |
VPRegionBlock * | createLoopRegion (const std::string &Name="", VPBlockBase *Entry=nullptr, VPBlockBase *Exiting=nullptr) |
Create a new loop region with Name and entry and exiting blocks set to Entry and Exiting respectively, if set. | |
VPRegionBlock * | createReplicateRegion (VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="") |
Create a new replicate region with Entry , Exiting and Name . | |
VPIRBasicBlock * | createEmptyVPIRBasicBlock (BasicBlock *IRBB) |
Create a VPIRBasicBlock wrapping IRBB , but do not create VPIRInstructions wrapping the instructions in tIRBB . | |
LLVM_ABI_FOR_TEST VPIRBasicBlock * | createVPIRBasicBlock (BasicBlock *IRBB) |
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB , except its terminator which is managed by the successors of the block in VPlan. | |
bool | hasEarlyExit () const |
Returns true if the VPlan is based on a loop with an early exit. | |
bool | hasScalarTail () const |
Returns true if the scalar tail may execute after the vector loop. |
Friends | |
class | VPlanPrinter |
class | VPSlotTracker |
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient output IR, including which branches, basic-blocks and output IR instructions to generate, and their cost.
VPlan holds a Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry VPBasicBlock.
VPlan::VPlan | ( | Loop * | L | ) |
Construct a VPlan for L
.
This will create VPIRBasicBlocks wrapping the original preheader and scalar header of L
, to be used as entry and scalar header blocks of the new VPlan.
Definition at line 873 of file VPlan.cpp.
References createVPIRBasicBlock(), and setEntry().
|
inline |
Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock wrapping ScalarHeaderBB
and a trip count of TC
.
Definition at line 4195 of file VPlan.h.
References createVPBasicBlock(), createVPIRBasicBlock(), and setEntry().
VPlan::~VPlan | ( | ) |
Definition at line 883 of file VPlan.cpp.
References llvm::dyn_cast(), getLiveIns(), and I.
Definition at line 4434 of file VPlan.h.
References assert().
Referenced by llvm::vputils::getOrCreateVPValueForSCEVExpr().
|
inline |
InstructionCost VPlan::cost | ( | ElementCount | VF, |
VPCostContext & | Ctx ) |
Return the cost of this plan.
Definition at line 1009 of file VPlan.cpp.
References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::VPBasicBlock::cost(), llvm::VPRegionBlock::cost(), getEntry(), llvm::InstructionCost::getInvalid(), getVectorLoopRegion(), llvm::InstructionCost::isValid(), and llvm::vp_depth_first_shallow().
VPIRBasicBlock * VPlan::createEmptyVPIRBasicBlock | ( | BasicBlock * | IRBB | ) |
Create a VPIRBasicBlock wrapping IRBB
, but do not create VPIRInstructions wrapping the instructions in tIRBB
.
The returned block is owned by the VPlan and deleted once the VPlan is destroyed.
Definition at line 1237 of file VPlan.cpp.
Referenced by llvm::VPIRBasicBlock::clone(), and createVPIRBasicBlock().
|
inline |
Create a new loop region with Name
and entry and exiting blocks set to Entry
and Exiting
respectively, if set.
The returned block is owned by the VPlan and deleted once the VPlan is destroyed.
Definition at line 4456 of file VPlan.h.
Referenced by createLoopRegion().
|
inline |
Create a new replicate region with Entry
, Exiting
and Name
.
The returned block is owned by the VPlan and deleted once the VPlan is destroyed.
Definition at line 4468 of file VPlan.h.
Referenced by createReplicateRegion().
|
inline |
Create a new VPBasicBlock with Name
and containing Recipe
if present.
The returned block is owned by the VPlan and deleted once the VPlan is destroyed.
Definition at line 4446 of file VPlan.h.
Referenced by addInitialSkeleton(), llvm::VPBasicBlock::clone(), createReplicateRegion(), llvm::VPlanTransforms::handleUncountableEarlyExit(), llvm::VPBasicBlock::splitAt(), and VPlan().
VPIRBasicBlock * VPlan::createVPIRBasicBlock | ( | BasicBlock * | IRBB | ) |
Create a VPIRBasicBlock from IRBB
containing VPIRInstructions for all instructions in IRBB
, except its terminator which is managed by the successors of the block in VPlan.
The returned block is owned by the VPlan and deleted once the VPlan is destroyed.
Definition at line 1243 of file VPlan.cpp.
References llvm::BasicBlock::begin(), llvm::VPIRInstruction::create(), createEmptyVPIRBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::BasicBlock::getTerminator(), I, and llvm::make_range().
Referenced by llvm::VPlanTransforms::attachCheckBlock(), duplicate(), VPlan(), and VPlan().
LLVM_DUMP_METHOD void VPlan::dump | ( | ) | const |
Dump the plan to stderr (for debugging).
Definition at line 1128 of file VPlan.cpp.
References llvm::dbgs(), and print().
Referenced by execute().
VPlan * VPlan::duplicate | ( | ) |
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clones, and return it.
Definition at line 1169 of file VPlan.cpp.
References assert(), llvm::cast(), cloneFrom(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), createVPIRBasicBlock(), llvm::dyn_cast(), llvm::find_if(), llvm::VPIRBasicBlock::getIRBasicBlock(), getLiveIns(), llvm::VPBlockBase::getNumSuccessors(), getScalarHeader(), I, llvm::isa(), remapOperands(), llvm::seq(), and llvm::vp_depth_first_shallow().
Referenced by llvm::VPlanTransforms::narrowInterleaveGroups(), and llvm::LoopVectorizePass::processLoop().
void VPlan::execute | ( | VPTransformState * | State | ) |
Generate the IR code for this VPlan.
Generate the code inside the preheader and body of the vectorized loop.
Assumes a single pre-header basic-block was created for this. Introduce additional basic-blocks as needed, and fill them all.
Definition at line 921 of file VPlan.cpp.
References llvm::Block, llvm::cast(), llvm::dbgs(), llvm::DominatorTreeBase< BasicBlock, false >::Delete, llvm::DeleteDeadBlocks(), dump(), llvm::LoopBase< BlockT, LoopT >::getBlocksVector(), getExitBlocks(), llvm::vputils::getFirstLoopHeader(), getScalarHeader(), getScalarPreheader(), llvm::BasicBlock::getSingleSuccessor(), llvm::BasicBlock::getTerminator(), getUF(), llvm::VPBlockBase::hasPredecessors(), llvm::isa(), LLVM_DEBUG, llvm::make_early_inc_range(), and setName().
Referenced by llvm::LoopVectorizationPlanner::executePlan().
|
inline |
Definition at line 4302 of file VPlan.h.
Referenced by llvm::vputils::isHeaderMask().
|
inline |
Definition at line 4317 of file VPlan.h.
References llvm::BasicBlock::getContext(), llvm::VPIRBasicBlock::getIRBasicBlock(), and getScalarHeader().
Referenced by llvm::VPlanTransforms::addBranchWeightToMiddleTerminator(), llvm::VPlanTransforms::addExplicitVectorLength(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addMinimumIterationCheck(), llvm::VPlanTransforms::addMinimumVectorEpilogueIterationCheck(), llvm::VPlanTransforms::attachCheckBlock(), llvm::VPlanTransforms::createInterleaveGroups(), expandVPWidenIntOrFpInduction(), getFalse(), getTrue(), optimizeVectorInductionWidthForTCAndVFUF(), transformRecipestoEVLRecipes(), llvm::VPlanTransforms::truncateToMinimalBitwidths(), and tryToReplaceALMWithWideALM().
|
inline |
Definition at line 4214 of file VPlan.h.
Referenced by addInitialSkeleton(), llvm::VPlanTransforms::addMinimumIterationCheck(), llvm::VPlanTransforms::addMinimumVectorEpilogueIterationCheck(), addReplicateRegions(), llvm::VPlanTransforms::canonicalizeEVLLoops(), connectEpilogueVectorLoop(), llvm::VPlanTransforms::convertToConcreteRecipes(), cost(), llvm::VPlanTransforms::createLoopRegions(), llvm::VPlanTransforms::cse(), llvm::VPlanTransforms::dissolveLoopRegions(), llvm::VPlanTransforms::dropPoisonGeneratingRecipes(), llvm::LoopVectorizationPlanner::executePlan(), llvm::VPlanTransforms::expandSCEVs(), llvm::DomTreeNodeTraits< VPBlockBase >::getEntryNode(), llvm::vputils::getFirstLoopHeader(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), getVectorLoopRegion(), getVectorLoopRegion(), hasScalarTail(), llvm::VPlanTransforms::materializeBroadcasts(), llvm::VPlanTransforms::materializePacksAndUnpacks(), mergeBlocksIntoPredecessors(), mergeReplicateRegionsIntoSuccessors(), preparePlanForEpilogueVectorLoop(), preparePlanForMainVectorLoop(), print(), llvm::VPlanTransforms::removeBranchOnConst(), llvm::VPlanTransforms::removeDeadRecipes(), removeRedundantExpandSCEVRecipes(), llvm::VPlanTransforms::replaceSymbolicStrides(), llvm::VPlanTransforms::replicateByVF(), llvm::VPBlockBase::setPlan(), llvm::VPlanTransforms::simplifyRecipes(), sinkScalarOperands(), and llvm::VPlanTransforms::unrollByUF().
|
inline |
VPIRBasicBlock * VPlan::getExitBlock | ( | BasicBlock * | IRBB | ) | const |
Return the VPIRBasicBlock corresponding to IRBB
.
IRBB
must be an exit block.
Definition at line 906 of file VPlan.cpp.
References assert(), llvm::find_if(), getExitBlocks(), and llvm::VPIRBasicBlock::getIRBasicBlock().
|
inline |
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition at line 4266 of file VPlan.h.
Referenced by calculateEarlyExitCost(), createExtractsForLiveOuts(), execute(), llvm::LoopVectorizationPlanner::executePlan(), getExitBlock(), llvm::VPlanTransforms::handleEarlyExits(), and llvm::VPlanTransforms::optimizeInductionExitUsers().
|
inline |
Return a VPValue wrapping i1 false.
Definition at line 4398 of file VPlan.h.
References getContext(), llvm::ConstantInt::getFalse(), and getOrAddLiveIn().
Referenced by llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addMinimumIterationCheck(), and simplifyRecipe().
Return the live-in VPValue for V
, if there is one or nullptr otherwise.
Definition at line 4404 of file VPlan.h.
Referenced by llvm::VPlanTransforms::replaceSymbolicStrides().
Return the list of live-in VPValues available in the VPlan.
Definition at line 4407 of file VPlan.h.
References llvm::all_of(), assert(), llvm::is_contained(), and P.
Referenced by duplicate(), llvm::VPlanTransforms::materializeBroadcasts(), and ~VPlan().
|
inline |
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar tail loop or the exit block from the loop latch.
If there is an early exit from the vector loop, the middle block conceptully has the early exit block as third successor, split accross 2 VPBBs. In that case, the second VPBB selects whether to execute the scalar tail loop or the exit bock. If the scalar tail loop or exit block are known to always execute, the middle block may branch directly to that block. This function cannot be called once the vector loop region has been removed.
Definition at line 4239 of file VPlan.h.
References assert(), llvm::cast(), getScalarPreheader(), llvm::VPBlockBase::getSingleSuccessor(), getVectorLoopRegion(), and llvm::is_contained().
Referenced by llvm::VPlanTransforms::addBranchWeightToMiddleTerminator(), llvm::VPlanTransforms::addExitUsersForFirstOrderRecurrences(), llvm::VPlanTransforms::addScalarResumePhis(), calculateEarlyExitCost(), getMiddleBlock(), llvm::VPlanTransforms::handleMaxMinNumReductions(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), llvm::VPlanTransforms::optimizeInductionExitUsers(), preparePlanForMainVectorLoop(), and llvm::LoopVectorizePass::processLoop().
|
inline |
Definition at line 4252 of file VPlan.h.
References getMiddleBlock().
std::string VPlan::getName | ( | ) | const |
Return a string with the name of the plan and the applicable VFs and UFs.
Definition at line 1098 of file VPlan.cpp.
References llvm::drop_begin().
Referenced by print().
Gets the live-in VPValue for V
or adds a new live-in (if none exists yet) for V
.
Definition at line 4377 of file VPlan.h.
References assert(), and llvm::VPValue::isLiveIn().
Referenced by llvm::VPlanTransforms::addActiveLaneMask(), addCanonicalIVRecipes(), llvm::VPlanTransforms::addExplicitVectorLength(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addMinimumIterationCheck(), llvm::VPlanTransforms::addMinimumVectorEpilogueIterationCheck(), addVPLaneMaskPhiAndUpdateExitBranch(), llvm::VPlanTransforms::attachCheckBlock(), llvm::VPlanTransforms::canonicalizeEVLLoops(), cloneForLane(), llvm::VPBuilder::createElementCount(), llvm::VPlanTransforms::createInterleaveGroups(), llvm::VPlanTransforms::expandSCEVs(), getFalse(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), getTrue(), legalizeAndOptimizeInductions(), llvm::VPlanTransforms::materializeBackedgeTakenCount(), llvm::VPlanTransforms::materializeVectorTripCount(), llvm::VPlanTransforms::materializeVFAndVFxUF(), llvm::VPlanTransforms::narrowInterleaveGroups(), optimizeEarlyExitInductionUser(), optimizeLatchExitInductionUser(), optimizeVectorInductionWidthForTCAndVFUF(), preparePlanForEpilogueVectorLoop(), llvm::VPlanTransforms::replaceSymbolicStrides(), simplifyRecipe(), transformRecipestoEVLRecipes(), llvm::VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(), and tryToReplaceALMWithWideALM().
|
inline |
The backedge taken count of the original loop.
Definition at line 4297 of file VPlan.h.
Referenced by llvm::VPlanTransforms::materializeBackedgeTakenCount(), llvm::VPlanTransforms::materializeBroadcasts(), and optimizeVectorInductionWidthForTCAndVFUF().
|
inline |
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition at line 4262 of file VPlan.h.
Referenced by addInitialSkeleton(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addScalarResumePhis(), duplicate(), execute(), getContext(), getScalarPreheader(), llvm::VPlanTransforms::handleEarlyExits(), preparePlanForMainVectorLoop(), llvm::VPlanTransforms::replicateByVF(), and simplifyRecipe().
|
inline |
Return the VPBasicBlock for the preheader of the scalar loop.
Definition at line 4257 of file VPlan.h.
References llvm::cast(), and getScalarHeader().
Referenced by llvm::VPlanTransforms::addExitUsersForFirstOrderRecurrences(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addScalarResumePhis(), llvm::VPlanTransforms::attachCheckBlock(), connectEpilogueVectorLoop(), execute(), llvm::LoopVectorizationPlanner::executePlan(), fixScalarResumeValuesFromBypass(), getMiddleBlock(), llvm::VPlanTransforms::handleMaxMinNumReductions(), hasScalarTail(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), preparePlanForMainVectorLoop(), and llvm::LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo().
Definition at line 4430 of file VPlan.h.
Referenced by llvm::vputils::getOrCreateVPValueForSCEVExpr().
|
inline |
Returns the symbolic UF of the vector loop region.
Definition at line 4312 of file VPlan.h.
Referenced by llvm::VPlanTransforms::materializeVFAndVFxUF(), and llvm::VPlanTransforms::narrowInterleaveGroups().
|
inline |
The trip count of the original loop.
Definition at line 4276 of file VPlan.h.
References assert().
Referenced by llvm::VPlanTransforms::addActiveLaneMask(), llvm::VPlanTransforms::addExplicitVectorLength(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addMinimumIterationCheck(), addVPLaneMaskPhiAndUpdateExitBranch(), llvm::VPlanTransforms::canonicalizeEVLLoops(), llvm::LoopVectorizationPlanner::executePlan(), llvm::VPlanTransforms::expandSCEVs(), isConditionTrueViaVFAndUF(), llvm::vputils::isHeaderMask(), llvm::VPlanTransforms::materializeBackedgeTakenCount(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), llvm::VPlanTransforms::materializeVectorTripCount(), llvm::VPlanTransforms::materializeVFAndVFxUF(), optimizeVectorInductionWidthForTCAndVFUF(), preparePlanForEpilogueVectorLoop(), llvm::VPlanTransforms::replaceSymbolicStrides(), and simplifyBranchConditionForVFAndUF().
|
inline |
Return a VPValue wrapping i1 true.
Definition at line 4392 of file VPlan.h.
References getContext(), getOrAddLiveIn(), and llvm::ConstantInt::getTrue().
Referenced by llvm::VPlanTransforms::addMinimumIterationCheck(), simplifyBranchConditionForVFAndUF(), and transformRecipestoEVLRecipes().
|
inline |
Definition at line 4355 of file VPlan.h.
References assert().
Referenced by llvm::VPlanTransforms::addBranchWeightToMiddleTerminator(), execute(), llvm::VPlanTransforms::materializeVFAndVFxUF(), llvm::VPlanTransforms::optimizeForVFAndUF(), and simplifyRecipe().
VPRegionBlock * VPlan::getVectorLoopRegion | ( | ) |
Returns the VPRegionBlock of the vector loop.
Definition at line 1027 of file VPlan.cpp.
References B(), llvm::dyn_cast(), getEntry(), and llvm::vp_depth_first_shallow().
Referenced by llvm::VPlanTransforms::addActiveLaneMask(), llvm::VPlanTransforms::addExitUsersForFirstOrderRecurrences(), llvm::VPlanTransforms::addExplicitVectorLength(), llvm::VPlanTransforms::addScalarResumePhis(), addVPLaneMaskPhiAndUpdateExitBranch(), llvm::VPlanTransforms::adjustFixedOrderRecurrences(), llvm::VPlanTransforms::attachCheckBlock(), llvm::LoopVectorizationPlanner::buildVPlans(), llvm::calculateRegisterUsageForPlan(), llvm::VPlanTransforms::clearReductionWrapFlags(), llvm::collectEphemeralRecipesForVPlan(), llvm::VPlanTransforms::convertToAbstractRecipes(), cost(), llvm::VPlanTransforms::createLoopRegions(), createScalarIVSteps(), expandVPWidenIntOrFpInduction(), expandVPWidenPointerInduction(), findHeaderMask(), getMiddleBlock(), llvm::vputils::getRecipesForUncountableExit(), getVectorPreheader(), llvm::VPlanTransforms::handleMaxMinNumReductions(), hasReplicatorRegion(), llvm::VPlanTransforms::introduceMasksAndLinearize(), isConditionTrueViaVFAndUF(), llvm::vputils::isHeaderMask(), legalizeAndOptimizeInductions(), licm(), llvm::VPlanTransforms::materializePacksAndUnpacks(), llvm::VPlanTransforms::narrowInterleaveGroups(), narrowToSingleScalarRecipes(), optimizeEarlyExitInductionUser(), optimizeVectorInductionWidthForTCAndVFUF(), planContainsAdditionalSimplifications(), preparePlanForEpilogueVectorLoop(), preparePlanForMainVectorLoop(), removeRedundantCanonicalIVs(), removeRedundantInductionCasts(), llvm::VPlanTransforms::replaceSymbolicStrides(), llvm::VPlanTransforms::replicateByVF(), scalarizeInstruction(), llvm::LoopVectorizationPlanner::selectInterleaveCount(), simplifyBlends(), simplifyBranchConditionForVFAndUF(), transformRecipestoEVLRecipes(), llvm::VPlanTransforms::truncateToMinimalBitwidths(), llvm::VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(), tryToReplaceALMWithWideALM(), llvm::VPlanTransforms::unrollByUF(), llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo(), llvm::VPTypeAnalysis::VPTypeAnalysis(), and willGenerateVectors().
const VPRegionBlock * VPlan::getVectorLoopRegion | ( | ) | const |
Definition at line 1035 of file VPlan.cpp.
References B(), llvm::dyn_cast(), getEntry(), and llvm::vp_depth_first_shallow().
|
inline |
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition at line 4219 of file VPlan.h.
References llvm::cast(), llvm::VPBlockBase::getSinglePredecessor(), and getVectorLoopRegion().
Referenced by addVPLaneMaskPhiAndUpdateExitBranch(), llvm::VPlanTransforms::attachCheckBlock(), llvm::LoopVectorizationPlanner::executePlan(), expandVPWidenIntOrFpInduction(), licm(), llvm::VPlanTransforms::materializeBroadcasts(), llvm::VPlanTransforms::narrowInterleaveGroups(), optimizeVectorInductionWidthForTCAndVFUF(), planContainsAdditionalSimplifications(), llvm::LoopVectorizePass::processLoop(), simplifyBranchConditionForVFAndUF(), and llvm::VPlanTransforms::truncateToMinimalBitwidths().
|
inline |
The vector trip count.
Definition at line 4305 of file VPlan.h.
Referenced by addCanonicalIVRecipes(), llvm::VPlanTransforms::addMiddleCheck(), llvm::VPlanTransforms::addScalarResumePhis(), llvm::calculateRegisterUsageForPlan(), llvm::VPlanTransforms::canonicalizeEVLLoops(), llvm::VPlanTransforms::handleMaxMinNumReductions(), isConditionTrueViaVFAndUF(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), llvm::VPlanTransforms::materializeVectorTripCount(), and preparePlanForMainVectorLoop().
|
inline |
Returns the VF of the vector loop region.
Definition at line 4308 of file VPlan.h.
Referenced by llvm::VPlanTransforms::createInterleaveGroups(), createScalarIVSteps(), createWidenInductionRecipes(), llvm::vputils::isHeaderMask(), llvm::VPlanTransforms::materializeVFAndVFxUF(), llvm::VPlanTransforms::narrowInterleaveGroups(), transformRecipestoEVLRecipes(), and llvm::VPlanTransforms::tryToConvertVPInstructionsToVPRecipes().
|
inline |
Returns VF * UF of the vector loop region.
Definition at line 4315 of file VPlan.h.
Referenced by addCanonicalIVRecipes(), llvm::VPlanTransforms::canonicalizeEVLLoops(), llvm::VPlanTransforms::materializeVectorTripCount(), llvm::VPlanTransforms::materializeVFAndVFxUF(), and transformRecipestoEVLRecipes().
|
inline |
Returns true if the VPlan is based on a loop with an early exit.
That is the case if the VPlan has either more than one exit block or a single exit block with multiple predecessors (one for the exit via the latch and one via the other early exit).
Definition at line 4490 of file VPlan.h.
References llvm::count_if(), and llvm::VPBlockBase::hasPredecessors().
Referenced by llvm::LoopVectorizationPlanner::computeBestVF(), llvm::LoopVectorizationPlanner::executePlan(), and llvm::LoopVectorizationPlanner::selectInterleaveCount().
|
inline |
Definition at line 4336 of file VPlan.h.
References llvm::any_of().
Referenced by expandVPWidenPointerInduction(), and legalizeAndOptimizeInductions().
|
inline |
Returns true if the scalar tail may execute after the vector loop.
Note that this relies on unneeded branches to the scalar tail loop being removed.
Definition at line 4500 of file VPlan.h.
References getEntry(), getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), and llvm::VPBlockBase::hasPredecessors().
Referenced by llvm::VPlanTransforms::handleMaxMinNumReductions(), and llvm::VPlanTransforms::materializeConstantVectorTripCount().
|
inline |
Definition at line 4346 of file VPlan.h.
References assert(), llvm::ElementCount::getFixed(), and hasVF().
Referenced by llvm::VPlanTransforms::addExplicitVectorLength(), legalizeAndOptimizeInductions(), llvm::VPlanTransforms::materializeBroadcasts(), llvm::VPlanTransforms::materializePacksAndUnpacks(), narrowToSingleScalarRecipes(), sinkScalarOperands(), and llvm::VPlanTransforms::truncateToMinimalBitwidths().
Definition at line 4353 of file VPlan.h.
Referenced by llvm::LoopVectorizationPlanner::executePlan(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), llvm::VPlanTransforms::optimizeForVFAndUF(), and setUF().
|
inline |
Definition at line 4335 of file VPlan.h.
Referenced by llvm::LoopVectorizationPlanner::executePlan(), hasScalarVFOnly(), llvm::VPlanTransforms::materializeConstantVectorTripCount(), llvm::VPlanTransforms::optimizeForVFAndUF(), removeVF(), and setVF().
bool VPlan::isExitBlock | ( | VPBlockBase * | VPBB | ) |
Returns true if VPBB
is an exit block.
Definition at line 914 of file VPlan.cpp.
References llvm::is_contained().
|
inline |
Returns true if the VPlan already has been unrolled, i.e.
it has a single concrete UF.
Definition at line 4368 of file VPlan.h.
Referenced by simplifyRecipe().
LLVM_DUMP_METHOD void VPlan::print | ( | raw_ostream & | O | ) | const |
Print this VPlan to O
.
Definition at line 1081 of file VPlan.cpp.
References llvm::Block, getEntry(), getName(), printLiveIns(), and VPSlotTracker.
Referenced by dump(), llvm::operator<<(), and llvm::LoopVectorizationPlanner::printPlans().
LLVM_DUMP_METHOD void VPlan::printDOT | ( | raw_ostream & | O | ) | const |
Print this VPlan in DOT format to O
.
Definition at line 1122 of file VPlan.cpp.
References Printer, and VPlanPrinter.
Referenced by llvm::LoopVectorizationPlanner::printPlans().
void VPlan::printLiveIns | ( | raw_ostream & | O | ) | const |
Print the live-ins of this VPlan to O
.
Definition at line 1043 of file VPlan.cpp.
References VPSlotTracker.
Referenced by print().
|
inline |
|
inline |
Resets the trip count for the VPlan.
The caller must make sure all uses of the original trip count have been replaced.
Definition at line 4290 of file VPlan.h.
References assert(), and llvm::VPValue::getNumUsers().
Referenced by llvm::VPlanTransforms::expandSCEVs(), preparePlanForEpilogueVectorLoop(), and llvm::VPlanTransforms::replaceSymbolicStrides().
|
inline |
Definition at line 4203 of file VPlan.h.
References llvm::VPBlockBase::setPlan().
|
inline |
Set the trip count assuming it is currently null; if it is not - use resetTripCount().
Definition at line 4283 of file VPlan.h.
References assert().
Referenced by addInitialSkeleton().
|
inline |
Definition at line 4360 of file VPlan.h.
References assert(), and hasUF().
Referenced by llvm::VPlanTransforms::addExplicitVectorLength(), and llvm::VPlanTransforms::unrollByUF().
|
inline |
Definition at line 4329 of file VPlan.h.
References assert(), and hasVF().
Referenced by llvm::VPlanTransforms::narrowInterleaveGroups(), and llvm::VPlanTransforms::optimizeForVFAndUF().
|
inline |
Returns an iterator range over all VFs of the plan.
Definition at line 4342 of file VPlan.h.
Referenced by llvm::LoopVectorizationPlanner::computeBestVF(), and llvm::VPlanTransforms::narrowInterleaveGroups().
|
friend |
Definition at line 4113 of file VPlan.h.
References VPlanPrinter.
Referenced by printDOT(), and VPlanPrinter.
|
friend |
Definition at line 4114 of file VPlan.h.
References VPSlotTracker.
Referenced by print(), printLiveIns(), and VPSlotTracker.