VPlanTransforms.cpp File Reference

This file implements a set of utility VPlan to VPlan transformations. More...

#include "VPlanTransforms.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Metadata.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"

Go to the source code of this file.

Classes
class	SinkStoreInfo
	Helper for extra no-alias checks via known-safe recipe and SCEV. More...
struct	EarlyExitInfo

Functions
static std::optional< int64_t >	getConstantStride (VPValue *Addr, Type *AccessTy, PredicatedScalarEvolution &PSE, const Loop *L)
	If the pointer operand Addr of a memory access is an affine AddRec w.r.t.
static bool	canHoistOrSinkWithNoAliasCheck (const MemoryLocation &MemLoc, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, std::optional< SinkStoreInfo > SinkInfo={})
	Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.
static Type *	getLoadStoreValueType (VPReplicateRecipe *R, bool IsLoad)
	Get the value type of the replicate load or store.
template<unsigned Opcode>
static SmallVector< SmallVector< VPReplicateRecipe *, 4 > >	collectGroupedReplicateMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L, function_ref< bool(VPReplicateRecipe *)> FilterFn)
	Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.
static bool	sinkScalarOperands (VPlan &Plan)
static VPBasicBlock *	getPredicatedThenBlock (VPRegionBlock *R)
	If R is a triangle region, return the 'then' block of the triangle.
static bool	mergeReplicateRegionsIntoSuccessors (VPlan &Plan)
static VPRegionBlock *	createReplicateRegion (VPReplicateRecipe *PredRecipe, VPRegionBlock *ParentRegion, VPlan &Plan)
static void	addReplicateRegions (VPlan &Plan)
static void	removeRedundantInductionCasts (VPlan &Plan)
	Remove redundant casts of inductions.
static void	legalizeAndOptimizeInductions (VPlan &Plan)
	Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
static VPWidenInductionRecipe *	getOptimizableIVOf (VPValue *VPV, PredicatedScalarEvolution &PSE)
	Check if VPV is an untruncated wide induction, either before or after the increment.
static VPValue *	optimizeEarlyExitInductionUser (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE)
	Attempts to optimize the induction variable exit values for users in the early exit block.
static VPValue *	tryToComputeEndValueForInduction (VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPValue *VectorTC)
	Compute the end value for WideIV, unless it is truncated.
static VPValue *	optimizeLatchExitInductionUser (VPlan &Plan, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, PredicatedScalarEvolution &PSE)
	Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
static VPValue *	optimizeLatchExitIVUserViaSCEV (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE, VPValue *ResumeTC, const Loop *L)
static void	removeRedundantExpandSCEVRecipes (VPlan &Plan)
	Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.
static bool	simplifyLogicalRecipe (VPSingleDefRecipe *Def, VPBuilder &Builder, bool CanCreateNewRecipe)
	Try to simplify logical and bitwise recipes in Def.
static void	simplifyRecipe (VPSingleDefRecipe *Def)
	Try to simplify VPSingleDefRecipe Def.
static void	reassociateHeaderMask (VPlan &Plan)
	Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.
static std::optional< Instruction::BinaryOps >	getUnmaskedDivRemOpcode (Intrinsic::ID ID)
static void	narrowToSingleScalarRecipes (VPlan &Plan)
static void	removeCommonBlendMask (VPBlendRecipe *Blend)
	Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.
static void	simplifyBlends (VPlan &Plan)
	Normalize and simplify VPBlendRecipes.
static bool	optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF)
	Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
static bool	isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Return true if Cond is known to be true for given BestVF and BestUF.
static bool	tryToReplaceALMWithWideALM (VPlan &Plan, ElementCount VF, unsigned UF)
	Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.
static bool	simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Try to simplify the branch condition of Plan.
static bool	cannotHoistOrSinkRecipe (VPRecipeBase &R, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, bool Sinking=false)
	Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.
static void	licm (VPlan &Plan)
	Move loop-invariant recipes out of the vector loop region in Plan.
static std::optional< VPValue * >	getRecipesForUncountableExit (SmallVectorImpl< VPInstruction * > &Recipes, VPBasicBlock *LatchVPBB)
	Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison.
static bool	handleUncountableExitsWithSideEffects (VPlan &Plan, SmallVectorImpl< EarlyExitInfo > &Exits, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC)
	Update Plan to mask memory operations in the loop based on whether the early exit is taken or not.
static VPExpressionRecipe *	tryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static VPExpressionRecipe *	tryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static void	tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
static VPIRMetadata	getCommonMetadata (ArrayRef< VPReplicateRecipe * > Recipes)
template<unsigned Opcode>
static SmallVector< SmallVector< VPReplicateRecipe *, 4 > >	collectComplementaryPredicatedMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L)
template<typename InstType>
static VPReplicateRecipe *	findRecipeWithMinAlign (ArrayRef< VPReplicateRecipe * > Group)
static bool	canSinkStoreWithNoAliasCheck (ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution &PSE, const Loop &L)
static bool	canNarrowLoad (VPSingleDefRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx, bool IsScalable)
	Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
static bool	canNarrowOps (ArrayRef< VPValue * > Ops, bool IsScalable)
static std::optional< ElementCount >	isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI)
	Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.
static bool	isAlreadyNarrow (VPValue *VPV)
	Returns true if VPValue is a narrow VPValue.
static VPValue *	narrowInterleaveGroupOp (ArrayRef< VPValue * > Members, SmallPtrSetImpl< VPValue * > &NarrowedOps, VPBasicBlock *Preheader)
static VPWidenIntOrFpInductionRecipe *	getExpressionIV (VPValue *V)
	Check if V is a binary expression of a widened IV and a loop-invariant value.
static VPValue *	cloneBinOpForScalarIV (VPWidenRecipe *BinOp, VPValue *ScalarIV, VPWidenIntOrFpInductionRecipe *WidenIV)
	Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.
static bool	areVFParamsOk (const VFInfo &Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution &PSE, const Loop *L)
	Returns true if Info's parameter kinds are compatible with Args.
static Function *	findVectorVariant (CallInst *CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution &PSE, const Loop *L)
	Find a vector variant of CI for VF, respecting MaskRequired.
static CallWideningDecision	decideCallWidening (VPInstruction &VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext &CostCtx)
	Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.

Detailed Description

This file implements a set of utility VPlan to VPlan transformations.

Definition in file VPlanTransforms.cpp.

Function Documentation

addReplicateRegions()

void addReplicateRegions ( VPlan & Plan )

static

Definition at line 569 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), createReplicateRegion(), llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getExiting(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::RegionBase< Tr >::getParent(), llvm::VPBlockBase::getParent(), llvm::Value::hasName(), llvm::VPBlockUtils::insertOnEdge(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPRegionBlock::setExiting(), llvm::VPBasicBlock::splitAt(), llvm::SplitBlock(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

areVFParamsOk()

bool areVFParamsOk ( const VFInfo & Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution & PSE, const Loop * L )

static

Returns true if Info's parameter kinds are compatible with Args.

Definition at line 5575 of file VPlanTransforms.cpp.

References llvm::all_of(), and llvm::PredicatedScalarEvolution::getSE().

Referenced by findVectorVariant().

canHoistOrSinkWithNoAliasCheck()

bool canHoistOrSinkWithNoAliasCheck ( const MemoryLocation & MemLoc, VPBasicBlock * FirstBB, VPBasicBlock * LastBB, std::optional< SinkStoreInfo > SinkInfo = {} )

static

Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.

If SinkInfo is std::nullopt, only recipes that may write to memory are checked (for load hoisting). Otherwise recipes that both read and write memory are checked, and SCEV is used to prove no-alias between the group leader and other replicate recipes (for store sinking).

Definition at line 250 of file VPlanTransforms.cpp.

Referenced by cannotHoistOrSinkRecipe(), canSinkStoreWithNoAliasCheck(), and llvm::VPlanTransforms::hoistPredicatedLoads().

canNarrowLoad()

bool canNarrowLoad ( VPSingleDefRecipe * WideMember0, unsigned OpIdx, VPValue * OpV, unsigned Idx, bool IsScalable )

static

Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.

V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group. A live-in or recipe defined outside the loop region can be converted, if it is the same across all lanes, or we can create a BuildVector for it.

Definition at line 3929 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), llvm::VPValue::getScalarType(), IR, llvm::VPValue::isDefinedOutsideLoopRegions(), and OpIdx.

Referenced by canNarrowOps().

canNarrowOps()

bool canNarrowOps ( ArrayRef< VPValue * > Ops, bool IsScalable )

static

Definition at line 3952 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), canNarrowLoad(), canNarrowOps(), llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), llvm::vputils::getOpcode(), llvm::isa(), OpIdx, P, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by canNarrowOps(), and llvm::VPlanTransforms::narrowInterleaveGroups().

cannotHoistOrSinkRecipe()

bool cannotHoistOrSinkRecipe ( VPRecipeBase & R, VPBasicBlock * FirstBB, VPBasicBlock * LastBB, bool Sinking = false )

static

Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.

When sinking, passing Sinking = true ensures that assumes aren't sunk.

Definition at line 2342 of file VPlanTransforms.cpp.

References canHoistOrSinkWithNoAliasCheck(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), llvm::vputils::getMemoryLocation(), llvm::isa(), llvm::PatternMatch::m_Intrinsic(), and llvm::SCEVPatternMatch::match().

Referenced by licm().

canSinkStoreWithNoAliasCheck()

bool canSinkStoreWithNoAliasCheck ( ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution & PSE, const Loop & L )

static

Definition at line 3855 of file VPlanTransforms.cpp.

References llvm::ArrayRef< T >::back(), canHoistOrSinkWithNoAliasCheck(), llvm::ArrayRef< T >::front(), and llvm::vputils::getMemoryLocation().

Referenced by llvm::VPlanTransforms::sinkPredicatedStores().

cloneBinOpForScalarIV()

VPValue * cloneBinOpForScalarIV ( VPWidenRecipe * BinOp, VPValue * ScalarIV, VPWidenIntOrFpInductionRecipe * WidenIV )

static

Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.

Definition at line 4432 of file VPlanTransforms.cpp.

References assert(), llvm::VPWidenRecipe::clone(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPWidenRecipe::getOpcode(), llvm::VPRecipeBase::insertAfter(), and llvm::Instruction::isBinaryOp().

Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().

collectComplementaryPredicatedMemOps()

template<unsigned Opcode>

SmallVector< SmallVector< VPReplicateRecipe *, 4 > > collectComplementaryPredicatedMemOps ( VPlan & Plan, PredicatedScalarEvolution & PSE, const Loop * L )

static

Definition at line 3740 of file VPlanTransforms.cpp.

References llvm::all_equal(), assert(), llvm::bind_back(), collectGroupedReplicateMemOps(), Groups, llvm::VPReplicateRecipe::isPredicated(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::map_range(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

collectGroupedReplicateMemOps()

template<unsigned Opcode>

SmallVector< SmallVector< VPReplicateRecipe *, 4 > > collectGroupedReplicateMemOps ( VPlan & Plan, PredicatedScalarEvolution & PSE, const Loop * L, function_ref< bool(VPReplicateRecipe *)> FilterFn )

static

Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.

Definition at line 288 of file VPlanTransforms.cpp.

References A(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), getLoadStoreValueType(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getVectorLoopRegion(), Groups, llvm::isa(), llvm::VPDominatorTree::properlyDominates(), llvm::stable_sort(), llvm::to_vector(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::values(), and llvm::vp_depth_first_deep().

Referenced by collectComplementaryPredicatedMemOps().

createReplicateRegion()

VPRegionBlock * createReplicateRegion ( VPReplicateRecipe * PredRecipe, VPRegionBlock * ParentRegion, VPlan & Plan )

static

Definition at line 527 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createReplicateRegion(), llvm::VPlan::createVPBasicBlock(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPReplicateRecipe::getMask(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::DebugLoc::getUnknown(), llvm::VPBlockUtils::insertTwoBlocksAfter(), llvm::VPReplicateRecipe::isSingleScalar(), llvm::VPReplicateRecipe::operandsWithoutMask(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::user_empty().

Referenced by addReplicateRegions().

decideCallWidening()

CallWideningDecision decideCallWidening ( VPInstruction & VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext & CostCtx )

static

Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.

Definition at line 5637 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::VPReplicateRecipe::computeCallCost(), llvm::VPWidenCallRecipe::computeCallCost(), llvm::VPWidenIntrinsicRecipe::computeCallCost(), findVectorVariant(), llvm::InstructionCost::getInvalid(), llvm::VPValue::getLiveInIRValue(), llvm::VPInstruction::getNumOperandsWithoutMask(), llvm::VPUser::getOperand(), llvm::VPRecipeValue::getScalarType(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::getVectorIntrinsicIDForCall(), IntrinsicCost, llvm::VPCostContext::isFreeScalarIntrinsic(), llvm::VPCostContext::isMaskRequired(), llvm::ElementCount::isScalar(), llvm::VPCostContext::L, llvm::VPCostContext::PSE, llvm::VPCostContext::TLI, and llvm::VPCostContext::willBeScalarized().

Referenced by llvm::VPlanTransforms::makeCallWideningDecisions().

findRecipeWithMinAlign()

template<typename InstType>

VPReplicateRecipe * findRecipeWithMinAlign ( ArrayRef< VPReplicateRecipe * > Group )

static

Definition at line 3798 of file VPlanTransforms.cpp.

References A(), B(), llvm::cast(), and llvm::min_element().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

findVectorVariant()

Function * findVectorVariant ( CallInst * CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution & PSE, const Loop * L )

static

Find a vector variant of CI for VF, respecting MaskRequired.

Returns the variant function, or nullptr. Masked variants are assumed to take the mask as a trailing parameter.

Definition at line 5602 of file VPlanTransforms.cpp.

References areVFParamsOk(), llvm::find_if(), llvm::Module::getFunction(), llvm::VFDatabase::getMappings(), llvm::Instruction::getModule(), and llvm::CallBase::isNoBuiltin().

Referenced by decideCallWidening().

getCommonMetadata()

VPIRMetadata getCommonMetadata ( ArrayRef< VPReplicateRecipe * > Recipes )

static

Definition at line 3731 of file VPlanTransforms.cpp.

References llvm::drop_begin(), llvm::ArrayRef< T >::front(), and llvm::VPIRMetadata::intersect().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

getConstantStride()

std::optional< int64_t > getConstantStride ( VPValue * Addr, Type * AccessTy, PredicatedScalarEvolution & PSE, const Loop * L )

static

If the pointer operand Addr of a memory access is an affine AddRec w.r.t.

L with a constant stride, return the stride in units of AccessTy. Otherwise return std::nullopt.

Definition at line 49 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::vputils::getSCEVExprForVPValue(), and llvm::getStrideFromAddRec().

Referenced by llvm::VPlanTransforms::makeMemOpWideningDecisions(), and llvm::VPlanTransforms::tryToConvertVPInstructionsToVPRecipes().

getExpressionIV()

VPWidenIntOrFpInductionRecipe * getExpressionIV ( VPValue * V )

static

Check if V is a binary expression of a widened IV and a loop-invariant value.

Returns the widened IV if found, nullptr otherwise.

Definition at line 4413 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::VPValue::isDefinedOutsideLoopRegions(), llvm::Instruction::isIntDivRem(), and std::swap().

Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().

getLoadStoreValueType()

Type * getLoadStoreValueType ( VPReplicateRecipe * R, bool IsLoad )

static

Get the value type of the replicate load or store.

IsLoad indicates whether it is a load.

Definition at line 279 of file VPlanTransforms.cpp.

References llvm::Type::getScalarType().

Referenced by collectGroupedReplicateMemOps().

getOptimizableIVOf()

VPWidenInductionRecipe * getOptimizableIVOf ( VPValue * VPV, PredicatedScalarEvolution & PSE )

static

Check if VPV is an untruncated wide induction, either before or after the increment.

If so return the header IV (before the increment), otherwise return null.

Definition at line 820 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm_unreachable, llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_FAdd(), llvm::VPlanPatternMatch::m_GetElementPtr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().

getPredicatedThenBlock()

VPBasicBlock * getPredicatedThenBlock ( VPRegionBlock * R )

static

If R is a triangle region, return the 'then' block of the triangle.

Definition at line 416 of file VPlanTransforms.cpp.

References llvm::cast(), and llvm::dyn_cast().

Referenced by mergeReplicateRegionsIntoSuccessors().

getRecipesForUncountableExit()

std::optional< VPValue * > getRecipesForUncountableExit ( SmallVectorImpl< VPInstruction * > & Recipes, VPBasicBlock * LatchVPBB )

static

Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison.

The recipes are stored in Recipes.

Definition at line 2945 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPBasicBlock::getTerminator(), I, llvm::VPlanPatternMatch::m_AnyOf(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_ICmp(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::none_of(), llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by handleUncountableExitsWithSideEffects().

getUnmaskedDivRemOpcode()

std::optional< Instruction::BinaryOps > getUnmaskedDivRemOpcode ( Intrinsic::ID ID )

static

Definition at line 1714 of file VPlanTransforms.cpp.

Referenced by narrowToSingleScalarRecipes().

handleUncountableExitsWithSideEffects()

bool handleUncountableExitsWithSideEffects ( VPlan & Plan, SmallVectorImpl< EarlyExitInfo > & Exits, VPBasicBlock * HeaderVPBB, VPBasicBlock * LatchVPBB, VPBasicBlock * MiddleVPBB, Loop * TheLoop, PredicatedScalarEvolution & PSE, DominatorTree & DT, AssumptionCache * AC )

static

Update Plan to mask memory operations in the loop based on whether the early exit is taken or not.

We're currently expecting to find a loop with properties similar to the following:

for.body: ir<indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0> EMIT ir<arrayidx> = getelementptr inbounds nuw ir< >, ir<indvars.iv> EMIT-SCALAR ir<%0> = load ir<arrayidx> EMIT ir<cmp1> = icmp sgt ir<%0>, ir<5> EMIT vp<%1> = masked-cond ir<cmp1> Successor(s): if.end

if.end: EMIT ir<arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<indvars.iv> EMIT-SCALAR ir<%2> = load ir<arrayidx3> EMIT ir<add> = add nsw ir<%2>, ir<42> EMIT ir<arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<indvars.iv> EMIT store ir<add>, ir<arrayidx5> EMIT ir<indvars.iv.next> = add nuw nsw ir<indvars.iv>, ir<1> EMIT vp<%3> = any-of ir<%1> EMIT ir<exitcond.not> = icmp eq ir<indvars.iv.next>, ir<10000> EMIT branch-on-two-conds vp<%3>, ir<exitcond.not> Successor(s): middle.block, middle.block, for.body

We currently expect LoopVectorizationLegality to ensure that:

• There must also be a counted exit. We will need to support speculative or first-faulting loads before we can remove this restriction.
• Any stores within the loop must not alias with the load used for the uncountable exit. We can relax this a bit with runtime aliasing checks.
• Other memory operations in the loop can take place before or after the uncountable exit, but must also be unconditional. We need to support combining the conditions in VPlanPredicator.
• The loop must have a single unconditional load contributing to the uncountable exit comparison, and the other term must be loop-invariant. Improving upon this requires work in getRecipesForUncountableExit to handle more complex recipe graphs.

Definition at line 3097 of file VPlanTransforms.cpp.

References _, llvm::VPInstruction::ActiveLaneMask, assert(), llvm::cast(), Cond, llvm::VPBuilder::createAdd(), llvm::VPBuilder::createFirstActiveLane(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createScalarZExtOrTrunc(), llvm::dwarf_linker::DebugLoc, llvm::VPBlockUtils::disconnectBlocks(), DL, llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::VPBasicBlock::end(), llvm::erase(), llvm::VPInstruction::ExtractLane, llvm::find_singleton(), llvm::VPBasicBlock::front(), llvm::ScalarEvolution::getConstant(), llvm::VPlan::getConstantInt(), llvm::VPlan::getDataLayout(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPUser::getOperand(), getRecipesForUncountableExit(), llvm::VPlan::getScalarPreheader(), llvm::VPRecipeValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::VPlan::getZero(), I, llvm::CmpInst::ICMP_EQ, llvm::is_contained(), llvm::isDereferenceableAndAlignedInLoop(), IV, llvm::Load, llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::range_size(), llvm::reverse(), llvm::VPUser::setOperand(), and llvm::vp_rpo_plain_cfg_loop_body().

Referenced by llvm::VPlanTransforms::handleUncountableEarlyExits().

isAlreadyNarrow()

bool isAlreadyNarrow ( VPValue * VPV )

static

Returns true if VPValue is a narrow VPValue.

Definition at line 4035 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), and llvm::isa().

Referenced by narrowInterleaveGroupOp(), and llvm::VPlanTransforms::narrowInterleaveGroups().

isConditionTrueViaVFAndUF()

bool isConditionTrueViaVFAndUF ( VPValue * Cond, VPlan & Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution & PSE )

static

Return true if Cond is known to be true for given BestVF and BestUF.

Definition at line 2001 of file VPlanTransforms.cpp.

References llvm::any_of(), assert(), llvm::CallingConv::C, Cond, llvm::VPRegionBlock::getCanonicalIV(), llvm::ScalarEvolution::getElementCount(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::isa(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by simplifyBranchConditionForVFAndUF().

isConsecutiveInterleaveGroup()

std::optional< ElementCount > isConsecutiveInterleaveGroup ( VPInterleaveRecipe * InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo & TTI )

static

Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.

The interleave group must also access the full vector width.

Definition at line 3991 of file VPlanTransforms.cpp.

References llvm::all_of(), assert(), llvm::VPDef::definedValues(), llvm::VPInterleaveBase::getInterleaveGroup(), llvm::VPInterleaveBase::getMask(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getScalarType(), llvm::VPInterleaveBase::getStoredValues(), llvm::VPDef::getVPValue(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::TargetTransformInfo::RGK_ScalableVector, and Size.

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

legalizeAndOptimizeInductions()

void legalizeAndOptimizeInductions ( VPlan & Plan )

static

Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.

If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.

Definition at line 725 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::vputils::collectUsersRecursively(), llvm::vputils::createScalarIVSteps(), llvm::VPBuilder::createSingleScalarOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::APInt::isNonNegative(), llvm::vputils::isSingleScalar(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Binary(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::reverse(), llvm::vputils::scalarizeVPWidenPointerInduction(), and Users.

Referenced by llvm::VPlanTransforms::optimize().

licm()

void licm ( VPlan & Plan )

static

Move loop-invariant recipes out of the vector loop region in Plan.

Definition at line 2363 of file VPlanTransforms.cpp.

References llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), cannotHoistOrSinkRecipe(), llvm::cast(), llvm::dyn_cast(), llvm::VPBasicBlock::end(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::make_early_inc_range(), llvm::VPDominatorTree::properlyDominates(), llvm::reverse(), and llvm::vp_depth_first_shallow().

mergeReplicateRegionsIntoSuccessors()

bool mergeReplicateRegionsIntoSuccessors ( VPlan & Plan )

static

Definition at line 438 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallPtrSetImplBase::empty(), llvm::VPlan::getEntry(), llvm::VPBasicBlock::getFirstNonPhi(), getPredicatedThenBlock(), llvm::VPBlockBase::getSingleSuccessor(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

narrowInterleaveGroupOp()

VPValue * narrowInterleaveGroupOp ( ArrayRef< VPValue * > Members, SmallPtrSetImpl< VPValue * > & NarrowedOps, VPBasicBlock * Preheader )

static

Definition at line 4046 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPInstruction::BuildVector, llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBuilder::createWidenLoad(), llvm::ArrayRef< T >::drop_front(), llvm::dyn_cast(), E(), llvm::ArrayRef< T >::front(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), isAlreadyNarrow(), N, narrowInterleaveGroupOp(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by narrowInterleaveGroupOp().

narrowToSingleScalarRecipes()

void narrowToSingleScalarRecipes ( VPlan & Plan )

static

Definition at line 1729 of file VPlanTransforms.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBuilder::createSingleScalarOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastLane, llvm::VPInstruction::ExtractLastPart, llvm::VPInstruction::ExtractPenultimateElement, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPlan::getEntry(), llvm::vputils::getOpcode(), llvm::DebugLoc::getUnknown(), getUnmaskedDivRemOpcode(), llvm::VPlan::hasScalarVFOnly(), llvm::is_contained(), llvm::isa(), llvm::vputils::isDeadRecipe(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::make_early_inc_range(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), Opc, llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::optimize().

optimizeEarlyExitInductionUser()

VPValue * optimizeEarlyExitInductionUser ( VPlan & Plan, VPValue * Op, PredicatedScalarEvolution & PSE )

static

Attempts to optimize the induction variable exit values for users in the early exit block.

Definition at line 877 of file VPlanTransforms.cpp.

References B(), llvm::cast(), DL, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), getOptimizableIVOf(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_FirstActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

optimizeLatchExitInductionUser()

VPValue * optimizeLatchExitInductionUser ( VPlan & Plan, VPValue * Op, DenseMap< VPValue *, VPValue * > & EndValues, PredicatedScalarEvolution & PSE )

static

Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.

Definition at line 961 of file VPlanTransforms.cpp.

References assert(), B(), llvm::cast(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), getOptimizableIVOf(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getZero(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm_unreachable, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

optimizeLatchExitIVUserViaSCEV()

VPValue * optimizeLatchExitIVUserViaSCEV ( VPlan & Plan, VPValue * Op, PredicatedScalarEvolution & PSE, VPValue * ResumeTC, const Loop * L )

static

Definition at line 1008 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), DL, llvm::VPlan::getConstantInt(), llvm::VPValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::DebugLoc::getUnknown(), llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::Type::isIntOrPtrTy(), llvm::Type::isPointerTy(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_AffineAddRec(), llvm::SCEVPatternMatch::m_SpecificLoop(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPSCEVExpander::tryToExpand().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

optimizeVectorInductionWidthForTCAndVFUF()

bool optimizeVectorInductionWidthForTCAndVFUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF )

static

Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.

Definition at line 1927 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::cast(), llvm::IntegerType::get(), llvm::APInt::getActiveBits(), llvm::VPlan::getBackedgeTakenCount(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getSingleUser(), llvm::VPlan::getTripCount(), llvm::VPWidenIntOrFpInductionRecipe::getTruncInst(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::VPlan::getZero(), llvm::VPValue::hasMoreThanOneUniqueUser(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::PowerOf2Ceil(), llvm::APIntOps::RoundingUDiv(), and llvm::APInt::UP.

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

reassociateHeaderMask()

void reassociateHeaderMask ( VPlan & Plan )

static

Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.

in optimizeEVLMasks.

Definition at line 1689 of file VPlanTransforms.cpp.

References llvm::append_range(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPRegionBlock::getHeaderMask(), llvm::VPlan::getVectorLoopRegion(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::VPValue::users(), users, X, and Y.

Referenced by llvm::VPlanTransforms::optimize().

removeCommonBlendMask()

void removeCommonBlendMask ( VPBlendRecipe * Blend )

static

Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.

Definition at line 1829 of file VPlanTransforms.cpp.

References llvm::VPValue::getDefiningRecipe(), llvm::VPBlendRecipe::getMask(), llvm::VPBlendRecipe::getNumIncomingValues(), llvm::VPUser::getOperand(), I, llvm::VPBlendRecipe::isNormalized(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPBlendRecipe::setMask().

Referenced by simplifyBlends().

removeRedundantExpandSCEVRecipes()

void removeRedundantExpandSCEVRecipes ( VPlan & Plan )

static

Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.

Definition at line 1100 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getTripCount(), llvm::make_early_inc_range(), llvm::VPlan::resetTripCount(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().

Referenced by llvm::VPlanTransforms::optimize().

removeRedundantInductionCasts()

void removeRedundantInductionCasts ( VPlan & Plan )

static

Remove redundant casts of inductions.

Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.

Definition at line 648 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::optimize().

simplifyBlends()

void simplifyBlends ( VPlan & Plan )

static

Normalize and simplify VPBlendRecipes.

Should be run after simplifyRecipes to make sure the masks are simplified.

Simplify BLEND a, b, Not(mask) -> BLEND b, a, mask.

Definition at line 1846 of file VPlanTransforms.cpp.

References llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::cast_or_null(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPlanPatternMatch::m_False(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::vputils::recursivelyDeleteDeadRecipes(), removeCommonBlendMask(), llvm::VPValue::replaceAllUsesWith(), llvm::SmallPtrSetImplBase::size(), llvm::VPValue::user_empty(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimize().

simplifyBranchConditionForVFAndUF()

bool simplifyBranchConditionForVFAndUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution & PSE )

static

Try to simplify the branch condition of Plan.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 2134 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, llvm::CallingConv::C, Cond, llvm::ScalarEvolution::getElementCount(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_ULE, llvm::isa(), isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::PatternMatch::m_Add(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

simplifyLogicalRecipe()

bool simplifyLogicalRecipe ( VPSingleDefRecipe * Def, VPBuilder & Builder, bool CanCreateNewRecipe )

static

Try to simplify logical and bitwise recipes in Def.

Definition at line 1122 of file VPlanTransforms.cpp.

References llvm::CallingConv::C, llvm::VPlan::getAllOnesValue(), llvm::VPlan::getFalse(), llvm::VPlan::getZero(), llvm::PatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_c_BinaryAnd(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_LogicalAnd(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_False(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_Not(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SelectLike(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), Select, X, and Y.

Referenced by simplifyRecipe().

simplifyRecipe()

void simplifyRecipe ( VPSingleDefRecipe * Def )

static

Try to simplify VPSingleDefRecipe Def.

Definition at line 1264 of file VPlanTransforms.cpp.

References A(), llvm::all_equal(), llvm::all_of(), llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::Broadcast, llvm::CallingConv::C, llvm::cast(), DL, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::APInt::exactLogBase2(), llvm::CmpInst::FCMP_UNO, llvm::VPlanPatternMatch::findUserOf(), llvm::APInt::getBitWidth(), llvm::VPlan::getConcreteUF(), llvm::VPlan::getDataLayout(), llvm::CmpInst::getInversePredicate(), llvm::VPUser::getOperand(), llvm::Type::getScalarSizeInBits(), llvm::VPDef::getVPSingleValue(), llvm::VPlan::getZero(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::APInt::isPowerOf2(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_Mul(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ComputeReductionResult(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_DerivedIV(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLane(), llvm::VPlanPatternMatch::m_ExtractLastPart(), llvm::VPlanPatternMatch::m_ExtractPenultimateElement(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::PatternMatch::m_Intrinsic(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificCmp(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_True(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_UDiv(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPIRValue(), llvm::VPlanPatternMatch::m_VPPhi(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::vputils::onlyFirstLaneUsed(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceAllUsesWith(), simplifyLogicalRecipe(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::to_vector(), llvm::vputils::tryToFoldLiveIns(), X, and Y.

Referenced by llvm::VPlanTransforms::simplifyRecipes().

sinkScalarOperands()

bool sinkScalarOperands ( VPlan & Plan )

static

Definition at line 323 of file VPlanTransforms.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), Changed, llvm::VPSingleDefRecipe::clone(), llvm::VPBuilder::createSingleScalarOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::VPlan::hasScalarVFOnly(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::make_filter_range(), llvm::VPRecipeBase::moveBefore(), llvm::VPUser::operands(), llvm::VPValue::replaceUsesWithIf(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::VPValue::users(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

tryToComputeEndValueForInduction()

VPValue * tryToComputeEndValueForInduction ( VPWidenInductionRecipe * WideIV, VPBuilder & VectorPHBuilder, VPValue * VectorTC )

static

Compute the end value for WideIV, unless it is truncated.

Creates a VPDerivedIVRecipe for non-canonical inductions.

Definition at line 927 of file VPlanTransforms.cpp.

References llvm::VPBuilder::createDerivedIV(), llvm::VPBuilder::createScalarCast(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

tryToCreateAbstractReductionRecipe()

void tryToCreateAbstractReductionRecipe ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.

Definition at line 3695 of file VPlanTransforms.cpp.

References assert(), llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().

Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().

tryToMatchAndCreateExtendedReduction()

VPExpressionRecipe * tryToMatchAndCreateExtendedReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created recipe must be decomposed to its constituent recipes before execution.

Definition at line 3476 of file VPlanTransforms.cpp.

References A(), assert(), llvm::cast(), CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::InstructionCost::getInvalid(), llvm::RecurrenceDescriptor::getOpcode(), llvm::Type::getScalarType(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatchHelpers::m_Isa(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Range, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToMatchAndCreateMulAccumulateReduction()

VPExpressionRecipe * tryToMatchAndCreateMulAccumulateReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))). reduce.fadd(fmul(ext(A), ext(B)))

Definition at line 3528 of file VPlanTransforms.cpp.

References A(), assert(), B(), llvm::canConstantBeExtended(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::RecurrenceDescriptor::getOpcode(), llvm::VPWidenCastRecipe::getOpcode(), llvm::VPUser::getOperand(), llvm::TargetTransformInfo::getPartialReductionExtendKind(), llvm::Type::getScalarType(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Mul, Range, llvm::Sub, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToReplaceALMWithWideALM()

bool tryToReplaceALMWithWideALM ( VPlan & Plan, ElementCount VF, unsigned UF )

static

Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.

This applies to the active lane mask instructions both in the loop and in the preheader. Incoming values of all ActiveLaneMaskPHIs are updated to use the new extracts from the first active lane mask, which has it's last operand (multiplier) set to UF.

Definition at line 2040 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::VPInstruction::ActiveLaneMask, llvm::all_of(), assert(), llvm::VPBasicBlock::back(), llvm::cast(), DL, llvm::dyn_cast(), llvm::EnableWideActiveLaneMask, llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::Type::getInt1Ty(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPRecipeBase::insertAfter(), llvm::ElementCount::isVector(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::not_equal_to().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().