VPlanTransforms.cpp File Reference

This file implements a set of utility VPlan to VPlan transformations. More...

#include "VPlanTransforms.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"

Go to the source code of this file.

Functions
static bool	sinkScalarOperands (VPlan &Plan)
VPValue *	getPredicatedMask (VPRegionBlock *R)
	If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
static VPBasicBlock *	getPredicatedThenBlock (VPRegionBlock *R)
	If R is a triangle region, return the 'then' block of the triangle.
static bool	mergeReplicateRegionsIntoSuccessors (VPlan &Plan)
static VPRegionBlock *	createReplicateRegion (VPReplicateRecipe *PredRecipe, VPlan &Plan)
static void	addReplicateRegions (VPlan &Plan)
static bool	mergeBlocksIntoPredecessors (VPlan &Plan)
	Remove redundant VPBasicBlocks by merging them into their predecessor if the predecessor has a single successor.
static void	removeRedundantInductionCasts (VPlan &Plan)
	Remove redundant casts of inductions.
static void	removeRedundantCanonicalIVs (VPlan &Plan)
	Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
static bool	isDeadRecipe (VPRecipeBase &R)
	Returns true if R is dead and can be removed.
static VPScalarIVStepsRecipe *	createScalarIVSteps (VPlan &Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, Instruction *TruncI, VPValue *StartV, VPValue *Step, DebugLoc DL, VPBuilder &Builder)
static SmallVector< VPUser * >	collectUsersRecursively (VPValue *V)
static void	legalizeAndOptimizeInductions (VPlan &Plan)
	Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
static VPWidenInductionRecipe *	getOptimizableIVOf (VPValue *VPV, ScalarEvolution &SE)
	Check if VPV is an untruncated wide induction, either before or after the increment.
static VPValue *	optimizeEarlyExitInductionUser (VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op, ScalarEvolution &SE)
	Attempts to optimize the induction variable exit values for users in the early exit block.
static VPValue *	optimizeLatchExitInductionUser (VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)
	Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
static void	removeRedundantExpandSCEVRecipes (VPlan &Plan)
	Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.
static void	recursivelyDeleteDeadRecipes (VPValue *V)
static Value *	tryToFoldLiveIns (const VPRecipeBase &R, unsigned Opcode, ArrayRef< VPValue * > Operands, const DataLayout &DL, VPTypeAnalysis &TypeInfo)
	Try to fold R using InstSimplifyFolder.
static void	simplifyRecipe (VPRecipeBase &R, VPTypeAnalysis &TypeInfo)
	Try to simplify recipe R.
static void	narrowToSingleScalarRecipes (VPlan &Plan)
static void	removeCommonBlendMask (VPBlendRecipe *Blend)
	Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.
static void	simplifyBlends (VPlan &Plan)
	Normalize and simplify VPBlendRecipes.
static bool	optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF)
	Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
static bool	isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, ScalarEvolution &SE)
	Return true if Cond is known to be true for given BestVF and BestUF.
static bool	tryToReplaceALMWithWideALM (VPlan &Plan, ElementCount VF, unsigned UF)
	Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.
static bool	simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Try to simplify the branch condition of Plan.
static bool	sinkRecurrenceUsersAfterPrevious (VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
	Sink users of FOR after the recipe defining the previous value Previous of the recurrence.
static bool	hoistPreviousBeforeFORUsers (VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
	Try to hoist Previous and its operands before all users of FOR.
static void	licm (VPlan &Plan)
	Move loop-invariant recipes out of the vector loop region in Plan.
static VPActiveLaneMaskPHIRecipe *	addVPLaneMaskPhiAndUpdateExitBranch (VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck)
static VPSingleDefRecipe *	findHeaderMask (VPlan &Plan)
	Collect the header mask with the pattern: (ICMP_ULE, WideCanonicalIV, backedge-taken-count) TODO: Introduce explicit recipe for header-mask instead of searching for the header-mask pattern manually.
static VPRecipeBase *	optimizeMaskToEVL (VPValue *HeaderMask, VPRecipeBase &CurRecipe, VPTypeAnalysis &TypeInfo, VPValue &AllOneMask, VPValue &EVL)
	Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.
static void	transformRecipestoEVLRecipes (VPlan &Plan, VPValue &EVL)
	Replace recipes with their EVL variants.
static void	expandVPWidenIntOrFpInduction (VPWidenIntOrFpInductionRecipe *WidenIVR, VPTypeAnalysis &TypeInfo)
	Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.
static void	expandVPWidenPointerInduction (VPWidenPointerInductionRecipe *R, VPTypeAnalysis &TypeInfo)
	Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.
static VPExpressionRecipe *	tryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static VPExpressionRecipe *	tryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static void	tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
static bool	canNarrowLoad (VPWidenRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx)
	Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
static bool	isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, unsigned VF, VPTypeAnalysis &TypeInfo, unsigned VectorRegWidth)
	Returns true if IR is a full interleave group with factor and number of members both equal to VF.
static bool	isAlreadyNarrow (VPValue *VPV)
	Returns true if VPValue is a narrow VPValue.

Variables
cl::opt< bool >	EnableWideActiveLaneMask ("enable-wide-lane-mask", cl::init(false), cl::Hidden, cl::desc("Enable use of wide get active lane mask instructions"))

Detailed Description

This file implements a set of utility VPlan to VPlan transformations.

Definition in file VPlanTransforms.cpp.

Function Documentation

addReplicateRegions()

void addReplicateRegions ( VPlan & Plan )

static

Definition at line 386 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), createReplicateRegion(), llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getExiting(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::RegionBase< Tr >::getParent(), llvm::VPBlockBase::getParent(), llvm::Value::hasName(), llvm::VPBlockUtils::insertOnEdge(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPRegionBlock::setExiting(), llvm::VPBasicBlock::splitAt(), llvm::SplitBlock(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

addVPLaneMaskPhiAndUpdateExitBranch()

VPActiveLaneMaskPHIRecipe * addVPLaneMaskPhiAndUpdateExitBranch ( VPlan & Plan, bool DataAndControlFlowWithoutRuntimeCheck )

static

Definition at line 2302 of file VPlanTransforms.cpp.

References llvm::VPInstruction::ActiveLaneMask, llvm::VPUser::addOperand(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::CalculateTripCountMinusVF, llvm::VPInstruction::CanonicalIVIncrementForPart, llvm::cast(), llvm::DataAndControlFlowWithoutRuntimeCheck, DL, llvm::VPRecipeBase::eraseFromParent(), llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPlan::getOrAddLiveIn(), llvm::VPCanonicalIVPHIRecipe::getScalarType(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), and llvm::VPlan::getVectorPreheader().

Referenced by llvm::VPlanTransforms::addActiveLaneMask().

canNarrowLoad()

bool canNarrowLoad ( VPWidenRecipe * WideMember0, unsigned OpIdx, VPValue * OpV, unsigned Idx )

static

Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.

V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group.

Definition at line 3941 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), IR, and OpIdx.

collectUsersRecursively()

SmallVector< VPUser * > collectUsersRecursively ( VPValue * V )

static

Definition at line 618 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::VPDef::definedValues(), llvm::from_range, I, llvm::isa(), and Users.

Referenced by llvm::VPlanTransforms::clearReductionWrapFlags(), legalizeAndOptimizeInductions(), and transformRecipestoEVLRecipes().

createReplicateRegion()

VPRegionBlock * createReplicateRegion ( VPReplicateRecipe * PredRecipe, VPlan & Plan )

static

Definition at line 343 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createVPBasicBlock(), llvm::VPlan::createVPRegionBlock(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPValue::getDefiningRecipe(), llvm::VPReplicateRecipe::getMask(), llvm::VPValue::getNumUsers(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::DebugLoc::getUnknown(), llvm::VPBlockUtils::insertTwoBlocksAfter(), llvm::VPReplicateRecipe::isSingleScalar(), llvm::make_range(), llvm::VPUser::op_begin(), llvm::VPUser::op_end(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPUser::setOperand().

Referenced by addReplicateRegions().

createScalarIVSteps()

VPScalarIVStepsRecipe * createScalarIVSteps ( VPlan & Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator * FPBinOp, Instruction * TruncI, VPValue * StartV, VPValue * Step, DebugLoc DL, VPBuilder & Builder )

static

Definition at line 580 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), DL, llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getScalarSizeInBits(), llvm::VPBlockBase::getSingleHierarchicalPredecessor(), llvm::Value::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), llvm::VPTypeAnalysis::inferScalarType(), and llvm::Type::isIntegerTy().

Referenced by legalizeAndOptimizeInductions().

expandVPWidenIntOrFpInduction()

void expandVPWidenIntOrFpInduction ( VPWidenIntOrFpInductionRecipe * WidenIVR, VPTypeAnalysis & TypeInfo )

static

Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.

In the following example:

vector.ph: Successor(s): vector loop

<x1> vector loop: { vector.body: WIDEN-INDUCTION i = phi start, step, vf ... EMIT branch-on-count ... No successors }

WIDEN-INDUCTION will get expanded to:

vector.ph: ... vp<induction.start> = ... vp<induction.increment> = ...

Successor(s): vector loop

<x1> vector loop: { vector.body: ir<i> = WIDEN-PHI vp<induction.start>, vp<vec.ind.next> ... vp<vec.ind.next> = add ir<i>, vp<induction.increment> EMIT branch-on-count ... No successors }

Definition at line 3128 of file VPlanTransforms.cpp.

References assert(), llvm::VPInstruction::Broadcast, DL, llvm::IntegerType::get(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPWidenIntOrFpInductionRecipe::getLastUnrolledPartOperand(), llvm::VPRecipeBase::getParent(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPBlockBase::getPlan(), llvm::Type::getScalarSizeInBits(), llvm::VPWidenIntOrFpInductionRecipe::getSplatVFValue(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPBasicBlock::getTerminator(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::InductionDescriptor::IK_IntInduction, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Next, llvm::VPValue::replaceAllUsesWith(), and llvm::VPInstruction::StepVector.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

expandVPWidenPointerInduction()

void expandVPWidenPointerInduction ( VPWidenPointerInductionRecipe * R, VPTypeAnalysis & TypeInfo )

static

Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.

In the following example:

<x1> vector loop: { vector.body: EMIT ir<ptr.iv> = WIDEN-POINTER-INDUCTION start, step, vf ... EMIT branch-on-count ... }

WIDEN-POINTER-INDUCTION will get expanded to:

<x1> vector loop: { vector.body: EMIT-SCALAR pointer.phi = phi start, ptr.ind EMIT mul = mul stepvector, step EMIT vector.gep = wide-ptradd pointer.phi, mul ... EMIT ptr.ind = ptradd pointer.phi, vf EMIT branch-on-count ... }

Definition at line 3240 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), DL, llvm::VPBlockBase::getExitingBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::InductionDescriptor::IK_PtrInduction, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isPointerTy(), llvm::Offset, llvm::VPInstruction::StepVector, and llvm::VPInstruction::WidePtrAdd.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

findHeaderMask()

VPSingleDefRecipe * findHeaderMask ( VPlan & Plan )

static

Collect the header mask with the pattern: (ICMP_ULE, WideCanonicalIV, backedge-taken-count) TODO: Introduce explicit recipe for header-mask instead of searching for the header-mask pattern manually.

Definition at line 2380 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::count_if(), llvm::dyn_cast(), llvm::iterator_range< IteratorT >::end(), llvm::find_if(), llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::IsaPred, llvm::vputils::isHeaderMask(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::addActiveLaneMask(), and transformRecipestoEVLRecipes().

getOptimizableIVOf()

VPWidenInductionRecipe * getOptimizableIVOf ( VPValue * VPV, ScalarEvolution & SE )

static

Check if VPV is an untruncated wide induction, either before or after the increment.

If so return the header IV (before the increment), otherwise return null.

Definition at line 724 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm_unreachable, llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_c_Binary(), llvm::VPlanPatternMatch::m_GetElementPtr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().

getPredicatedMask()

VPValue * getPredicatedMask

(

VPRegionBlock *

R

)

If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.

Definition at line 222 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::dyn_cast(), and llvm::isa().

Referenced by mergeReplicateRegionsIntoSuccessors().

getPredicatedThenBlock()

VPBasicBlock * getPredicatedThenBlock ( VPRegionBlock * R )

static

If R is a triangle region, return the 'then' block of the triangle.

Definition at line 232 of file VPlanTransforms.cpp.

References llvm::cast(), and llvm::dyn_cast().

Referenced by mergeReplicateRegionsIntoSuccessors().

hoistPreviousBeforeFORUsers()

bool hoistPreviousBeforeFORUsers ( VPFirstOrderRecurrencePHIRecipe * FOR, VPRecipeBase * Previous, VPDominatorTree & VPDT )

static

Try to hoist Previous and its operands before all users of FOR.

Definition at line 1805 of file VPlanTransforms.cpp.

References A(), llvm::all_of(), assert(), B(), llvm::cast(), llvm::VPBasicBlock::getEnclosingLoopRegion(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPDef::getNumDefinedValues(), llvm::VPBlockBase::getParent(), llvm::VPRecipeBase::getParent(), llvm::VPDef::getVPSingleValue(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::VPRecipeBase::mayHaveSideEffects(), llvm::VPRecipeBase::mayReadFromMemory(), llvm::VPUser::operands(), llvm::VPDominatorTree::properlyDominates(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::sort().

Referenced by llvm::VPlanTransforms::adjustFixedOrderRecurrences().

isAlreadyNarrow()

bool isAlreadyNarrow ( VPValue * VPV )

static

Returns true if VPValue is a narrow VPValue.

Definition at line 3988 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), and llvm::VPValue::isLiveIn().

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

isConditionTrueViaVFAndUF()

bool isConditionTrueViaVFAndUF ( VPValue * Cond, VPlan & Plan, ElementCount BestVF, unsigned BestUF, ScalarEvolution & SE )

static

Return true if Cond is known to be true for given BestVF and BestUF.

Definition at line 1515 of file VPlanTransforms.cpp.

References llvm::any_of(), assert(), llvm::CallingConv::C, Cond, llvm::VPlan::getCanonicalIV(), llvm::ScalarEvolution::getElementCount(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), llvm::VPlan::getVectorTripCount(), llvm::CmpInst::ICMP_EQ, llvm::isa(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by simplifyBranchConditionForVFAndUF().

isConsecutiveInterleaveGroup()

bool isConsecutiveInterleaveGroup ( VPInterleaveRecipe * InterleaveR, unsigned VF, VPTypeAnalysis & TypeInfo, unsigned VectorRegWidth )

static

Returns true if IR is a full interleave group with factor and number of members both equal to VF.

The interleave group must also access the full vector width VectorRegWidth.

Definition at line 3957 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPDef::definedValues(), llvm::VPInterleaveBase::getInterleaveGroup(), llvm::Type::getScalarSizeInBits(), llvm::VPInterleaveBase::getStoredValues(), llvm::VPDef::getVPValue(), and llvm::VPTypeAnalysis::inferScalarType().

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

isDeadRecipe()

bool isDeadRecipe ( VPRecipeBase & R )

static

Returns true if R is dead and can be removed.

Definition at line 536 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::PatternMatch::m_Intrinsic(), and llvm::PatternMatch::match().

Referenced by recursivelyDeleteDeadRecipes(), and llvm::VPlanTransforms::removeDeadRecipes().

legalizeAndOptimizeInductions()

void legalizeAndOptimizeInductions ( VPlan & Plan )

static

Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.

If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.

Definition at line 642 of file VPlanTransforms.cpp.

References llvm::cast(), collectUsersRecursively(), createScalarIVSteps(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::InductionDescriptor::IK_IntInduction, llvm::isa(), llvm::vputils::isSingleScalar(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and Users.

Referenced by llvm::VPlanTransforms::optimize().

licm()

void licm ( VPlan & Plan )

static

Move loop-invariant recipes out of the vector loop region in Plan.

Definition at line 2092 of file VPlanTransforms.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPBasicBlock::end(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::make_early_inc_range(), and llvm::vp_depth_first_shallow().

mergeBlocksIntoPredecessors()

bool mergeBlocksIntoPredecessors ( VPlan & Plan )

static

Remove redundant VPBasicBlocks by merging them into their predecessor if the predecessor has a single successor.

Definition at line 418 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPBasicBlock::end(), llvm::VPlan::getEntry(), llvm::isa(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::to_vector(), and llvm::vp_depth_first_deep().

mergeReplicateRegionsIntoSuccessors()

bool mergeReplicateRegionsIntoSuccessors ( VPlan & Plan )

static

Definition at line 254 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallPtrSetImplBase::empty(), llvm::VPlan::getEntry(), llvm::VPBasicBlock::getFirstNonPhi(), getPredicatedMask(), getPredicatedThenBlock(), llvm::VPBlockBase::getSingleSuccessor(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

narrowToSingleScalarRecipes()

void narrowToSingleScalarRecipes ( VPlan & Plan )

static

Definition at line 1292 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastElement, llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::vputils::isSingleScalar(), llvm::make_early_inc_range(), llvm::reverse(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimize().

optimizeEarlyExitInductionUser()

VPValue * optimizeEarlyExitInductionUser ( VPlan & Plan, VPTypeAnalysis & TypeInfo, VPBlockBase * PredVPBB, VPValue * Op, ScalarEvolution & SE )

static

Attempts to optimize the induction variable exit values for users in the early exit block.

Definition at line 782 of file VPlanTransforms.cpp.

References B(), llvm::cast(), DL, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPInstruction::FirstActiveLane, llvm::VPlan::getCanonicalIV(), getOptimizableIVOf(), llvm::VPlan::getOrAddLiveIn(), llvm::VPCanonicalIVPHIRecipe::getScalarType(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionExitUsers().

optimizeLatchExitInductionUser()

VPValue * optimizeLatchExitInductionUser ( VPlan & Plan, VPTypeAnalysis & TypeInfo, VPBlockBase * PredVPBB, VPValue * Op, DenseMap< VPValue *, VPValue * > & EndValues, ScalarEvolution & SE )

static

Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.

Definition at line 837 of file VPlanTransforms.cpp.

References assert(), B(), llvm::cast(), getOptimizableIVOf(), llvm::VPlan::getOrAddLiveIn(), llvm::DebugLoc::getUnknown(), llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm_unreachable, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::VPlanPatternMatch::m_ExtractLastElement(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionExitUsers().

optimizeMaskToEVL()

VPRecipeBase * optimizeMaskToEVL ( VPValue * HeaderMask, VPRecipeBase & CurRecipe, VPTypeAnalysis & TypeInfo, VPValue & AllOneMask, VPValue & EVL )

static

Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.

Returns nullptr if no EVL-based recipe could be created. HeaderMask Header Mask. CurRecipe Recipe to be transform. TypeInfo VPlan-based type analysis. AllOneMask The vector mask parameter of vector-predication intrinsics. EVL The explicit vector length parameter of vector-predication intrinsics.

Adjust any end pointers so that they point to the end of EVL lanes not VF.

Definition at line 2465 of file VPlanTransforms.cpp.

References llvm::all_of(), assert(), llvm::TypeSwitch< T, ResultT >::Case(), llvm::VPVectorEndPointerRecipe::clone(), llvm::Default, llvm::dyn_cast(), llvm::VPWidenMemoryRecipe::getAddr(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenMemoryRecipe::getMask(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertBefore(), IR, LHS, llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), RHS, and llvm::VPUser::setOperand().

Referenced by transformRecipestoEVLRecipes().

optimizeVectorInductionWidthForTCAndVFUF()

bool optimizeVectorInductionWidthForTCAndVFUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF )

static

Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.

Definition at line 1444 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), llvm::cast(), llvm::dyn_cast(), llvm::dyn_cast_if_present(), llvm::IntegerType::get(), llvm::APInt::getActiveBits(), llvm::APInt::getBitWidth(), llvm::VPlan::getContext(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPlan::getTripCount(), llvm::VPValue::getUnderlyingValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::VPValue::isLiveIn(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::PowerOf2Ceil(), llvm::APIntOps::RoundingUDiv(), and llvm::APInt::UP.

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

recursivelyDeleteDeadRecipes()

void recursivelyDeleteDeadRecipes ( VPValue * V )

static

Definition at line 926 of file VPlanTransforms.cpp.

References llvm::SmallVectorImpl< T >::append(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPValue::getDefiningRecipe(), llvm::SmallPtrSetImpl< PtrType >::insert(), isDeadRecipe(), llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by simplifyBlends(), and transformRecipestoEVLRecipes().

removeCommonBlendMask()

void removeCommonBlendMask ( VPBlendRecipe * Blend )

static

Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.

Definition at line 1346 of file VPlanTransforms.cpp.

References llvm::VPValue::getDefiningRecipe(), llvm::VPBlendRecipe::getMask(), llvm::VPBlendRecipe::getNumIncomingValues(), llvm::VPUser::getOperand(), I, llvm::VPBlendRecipe::isNormalized(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPBlendRecipe::setMask().

Referenced by simplifyBlends().

removeRedundantCanonicalIVs()

void removeRedundantCanonicalIVs ( VPlan & Plan )

static

Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.

Definition at line 503 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::vputils::onlyFirstLaneUsed(), llvm::vputils::onlyScalarValuesUsed(), llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::optimize().

removeRedundantExpandSCEVRecipes()

void removeRedundantExpandSCEVRecipes ( VPlan & Plan )

static

Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.

Definition at line 909 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::make_early_inc_range(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().

Referenced by llvm::VPlanTransforms::optimize().

removeRedundantInductionCasts()

void removeRedundantInductionCasts ( VPlan & Plan )

static

Remove redundant casts of inductions.

Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.

Definition at line 470 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::optimize().

simplifyBlends()

void simplifyBlends ( VPlan & Plan )

static

Normalize and simplify VPBlendRecipes.

Should be run after simplifyRecipes to make sure the masks are simplified.

Simplify BLEND a, b, Not(mask) -> BLEND b, a, mask.

Definition at line 1363 of file VPlanTransforms.cpp.

References llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::cast_or_null(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), llvm::VPValue::getNumUsers(), llvm::VPlan::getVectorLoopRegion(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPlanPatternMatch::m_False(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), recursivelyDeleteDeadRecipes(), removeCommonBlendMask(), llvm::VPValue::replaceAllUsesWith(), llvm::SmallPtrSetImplBase::size(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimize().

simplifyBranchConditionForVFAndUF()

bool simplifyBranchConditionForVFAndUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution & PSE )

static

Try to simplify the branch condition of Plan.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 1644 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPBasicBlock::appendRecipe(), assert(), B(), llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, llvm::CallingConv::C, llvm::cast(), Cond, llvm::VPBlockUtils::connectBlocks(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast(), llvm::ScalarEvolution::getElementCount(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::CmpInst::ICMP_ULE, llvm::isa(), isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::SCEV::isZero(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy(), llvm::VPValue::replaceAllUsesWith(), llvm::VPlanTransforms::simplifyRecipes(), llvm::VPInstruction::StepVector, and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

simplifyRecipe()

void simplifyRecipe ( VPRecipeBase & R, VPTypeAnalysis & TypeInfo )

static

Try to simplify recipe R.

Definition at line 999 of file VPlanTransforms.cpp.

References A(), llvm::all_of(), assert(), llvm::CallingConv::C, llvm::cast(), DL, llvm::dyn_cast(), llvm::BasicBlock::getDataLayout(), llvm::ConstantInt::getFalse(), llvm::CmpInst::getInversePredicate(), llvm::VPIRBasicBlock::getIRBasicBlock(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getScalarHeader(), llvm::Type::getScalarSizeInBits(), I, llvm::VPTypeAnalysis::inferScalarType(), llvm::isa(), llvm::Type::isIntegerTy(), llvm::vputils::isSingleScalar(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::VPlanPatternMatch::m_c_BinaryAnd(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_Mul(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_CombineOr(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_DerivedIV(), llvm::VPlanPatternMatch::m_ExtractLastElement(), llvm::VPlanPatternMatch::m_False(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_True(), llvm::PatternMatch::m_Trunc(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::vputils::onlyFirstLaneUsed(), llvm::to_vector(), tryToFoldLiveIns(), X, and Y.

Referenced by llvm::VPlanTransforms::simplifyRecipes().

sinkRecurrenceUsersAfterPrevious()

bool sinkRecurrenceUsersAfterPrevious ( VPFirstOrderRecurrencePHIRecipe * FOR, VPRecipeBase * Previous, VPDominatorTree & VPDT )

static

Sink users of FOR after the recipe defining the previous value Previous of the recurrence.

Returns

true if all users of FOR could be re-arranged as needed or false if it is not possible.

Definition at line 1750 of file VPlanTransforms.cpp.

References A(), assert(), B(), llvm::cast(), llvm::VPDef::getNumDefinedValues(), llvm::VPDef::getVPSingleValue(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::VPDominatorTree::properlyDominates(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::sort(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::adjustFixedOrderRecurrences().

sinkScalarOperands()

bool sinkScalarOperands ( VPlan & Plan )

static

Definition at line 133 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), Changed, llvm::VPSingleDefRecipe::clone(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::VPlan::hasScalarVFOnly(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::VPRecipeBase::mayHaveSideEffects(), llvm::VPRecipeBase::mayReadOrWriteMemory(), llvm::VPRecipeBase::moveBefore(), llvm::VPUser::operands(), llvm::VPValue::replaceUsesWithIf(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::VPValue::users(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

transformRecipestoEVLRecipes()

void transformRecipestoEVLRecipes ( VPlan & Plan, VPValue & EVL )

static

Replace recipes with their EVL variants.

Definition at line 2537 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), collectUsersRecursively(), findHeaderMask(), llvm::VPlan::getContext(), llvm::VPValue::getDefiningRecipe(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getInt32Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPDef::getNumDefinedValues(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrAddLiveIn(), llvm::VPRecipeBase::getParent(), llvm::VPRegionBlock::getPreheaderVPBB(), llvm::ConstantInt::getSigned(), llvm::VPlan::getTrue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), llvm::VPlan::getVFxUF(), llvm::VPDef::getVPValue(), I, llvm::CmpInst::ICMP_ULT, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::IsaPred, llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), optimizeMaskToEVL(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), recursivelyDeleteDeadRecipes(), llvm::VPValue::replaceAllUsesWith(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), llvm::VPInstruction::StepVector, llvm::VPValue::users(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::addExplicitVectorLength().

tryToCreateAbstractReductionRecipe()

void tryToCreateAbstractReductionRecipe ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.

Definition at line 3625 of file VPlanTransforms.cpp.

References llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().

Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().

tryToFoldLiveIns()

Value * tryToFoldLiveIns ( const VPRecipeBase & R, unsigned Opcode, ArrayRef< VPValue * > Operands, const DataLayout & DL, VPTypeAnalysis & TypeInfo )

static

Try to fold R using InstSimplifyFolder.

Will succeed and return a non-nullptr Value for a handled Opcode if corresponding Operands are foldable live-ins.

Definition at line 948 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::cast(), DL, llvm::drop_begin(), GEP, llvm::Constant::getAllOnesValue(), llvm::VPTypeAnalysis::getContext(), llvm::Type::getInt8Ty(), llvm::Constant::getNullValue(), getType(), llvm::VPTypeAnalysis::inferScalarType(), llvm::Instruction::isBinaryOp(), llvm::Instruction::isCast(), llvm::VPInstruction::LogicalAnd, llvm::VPInstruction::Not, Operands, llvm::VPInstruction::PtrAdd, and llvm::VPInstruction::WidePtrAdd.

Referenced by simplifyRecipe().

tryToMatchAndCreateExtendedReduction()

VPExpressionRecipe * tryToMatchAndCreateExtendedReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created recipe must be decomposed to its constituent recipes before execution.

Definition at line 3464 of file VPlanTransforms.cpp.

References A(), llvm::cast(), CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::RecurrenceDescriptor::getOpcode(), llvm::InstructionCost::isValid(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Range, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToMatchAndCreateMulAccumulateReduction()

VPExpressionRecipe * tryToMatchAndCreateMulAccumulateReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))).

Definition at line 3509 of file VPlanTransforms.cpp.

References A(), B(), llvm::cast(), CostKind, llvm::dyn_cast_if_present(), llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::RecurrenceDescriptor::getOpcode(), llvm::VPRecipeBase::insertBefore(), llvm::InstructionCost::isValid(), llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_SpecificInt(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Mul, Range, llvm::Sub, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToReplaceALMWithWideALM()

bool tryToReplaceALMWithWideALM ( VPlan & Plan, ElementCount VF, unsigned UF )

static

Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.

This applies to the active lane mask instructions both in the loop and in the preheader. Incoming values of all ActiveLaneMaskPHIs are updated to use the new extracts from the first active lane mask, which has it's last operand (multiplier) set to UF.

Definition at line 1552 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::VPInstruction::ActiveLaneMask, llvm::all_of(), assert(), llvm::VPBasicBlock::back(), llvm::cast(), DL, llvm::dyn_cast(), EnableWideActiveLaneMask, llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::Type::getInt1Ty(), llvm::Type::getInt64Ty(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getVectorLoopRegion(), llvm::ElementCount::isVector(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::PatternMatch::m_ConstantInt(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

Variable Documentation

EnableWideActiveLaneMask

cl::opt< bool > EnableWideActiveLaneMask("enable-wide-lane-mask", cl::init(false), cl::Hidden, cl::desc("Enable use of wide get active lane mask instructions"))	(	"enable-wide-lane-mask"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Enable use of wide get active lane mask instructions")	)

Referenced by tryToReplaceALMWithWideALM().