LLVM 22.0.0git
Functions
VPlanTransforms.cpp File Reference

This file implements a set of utility VPlan to VPlan transformations. More...

#include "VPlanTransforms.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"

Go to the source code of this file.

Functions

static bool sinkScalarOperands (VPlan &Plan)
 
VPValuegetPredicatedMask (VPRegionBlock *R)
 If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
 
static VPBasicBlockgetPredicatedThenBlock (VPRegionBlock *R)
 If R is a triangle region, return the 'then' block of the triangle.
 
static bool mergeReplicateRegionsIntoSuccessors (VPlan &Plan)
 
static VPRegionBlockcreateReplicateRegion (VPReplicateRecipe *PredRecipe, VPlan &Plan)
 
static void addReplicateRegions (VPlan &Plan)
 
static bool mergeBlocksIntoPredecessors (VPlan &Plan)
 Remove redundant VPBasicBlocks by merging them into their predecessor if the predecessor has a single successor.
 
static void removeRedundantInductionCasts (VPlan &Plan)
 Remove redundant casts of inductions.
 
static void removeRedundantCanonicalIVs (VPlan &Plan)
 Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
 
static bool isDeadRecipe (VPRecipeBase &R)
 Returns true if R is dead and can be removed.
 
static VPScalarIVStepsRecipecreateScalarIVSteps (VPlan &Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, Instruction *TruncI, VPValue *StartV, VPValue *Step, DebugLoc DL, VPBuilder &Builder)
 
static SmallVector< VPUser * > collectUsersRecursively (VPValue *V)
 
static void legalizeAndOptimizeInductions (VPlan &Plan)
 Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
 
static VPWidenInductionRecipegetOptimizableIVOf (VPValue *VPV, ScalarEvolution &SE)
 Check if VPV is an untruncated wide induction, either before or after the increment.
 
static VPValueoptimizeEarlyExitInductionUser (VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op, ScalarEvolution &SE)
 Attempts to optimize the induction variable exit values for users in the early exit block.
 
static VPValueoptimizeLatchExitInductionUser (VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)
 Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
 
static void removeRedundantExpandSCEVRecipes (VPlan &Plan)
 Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.
 
static void recursivelyDeleteDeadRecipes (VPValue *V)
 
static ValuetryToFoldLiveIns (const VPRecipeBase &R, unsigned Opcode, ArrayRef< VPValue * > Operands, const DataLayout &DL, VPTypeAnalysis &TypeInfo)
 Try to fold R using InstSimplifyFolder.
 
static void simplifyRecipe (VPRecipeBase &R, VPTypeAnalysis &TypeInfo)
 Try to simplify recipe R.
 
static void narrowToSingleScalarRecipes (VPlan &Plan)
 
static void simplifyBlends (VPlan &Plan)
 Normalize and simplify VPBlendRecipes.
 
static bool optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF)
 Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
 
static bool isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, ScalarEvolution &SE)
 Return true if Cond is known to be true for given BestVF and BestUF.
 
static bool simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
 Try to simplify the branch condition of Plan.
 
static bool sinkRecurrenceUsersAfterPrevious (VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
 Sink users of FOR after the recipe defining the previous value Previous of the recurrence.
 
static bool hoistPreviousBeforeFORUsers (VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
 Try to hoist Previous and its operands before all users of FOR.
 
static void licm (VPlan &Plan)
 Move loop-invariant recipes out of the vector loop region in Plan.
 
static VPActiveLaneMaskPHIRecipeaddVPLaneMaskPhiAndUpdateExitBranch (VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck)
 
static VPSingleDefRecipefindHeaderMask (VPlan &Plan)
 Collect the header mask with the pattern: (ICMP_ULE, WideCanonicalIV, backedge-taken-count) TODO: Introduce explicit recipe for header-mask instead of searching for the header-mask pattern manually.
 
static VPRecipeBaseoptimizeMaskToEVL (VPValue *HeaderMask, VPRecipeBase &CurRecipe, VPTypeAnalysis &TypeInfo, VPValue &AllOneMask, VPValue &EVL)
 Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.
 
static void transformRecipestoEVLRecipes (VPlan &Plan, VPValue &EVL)
 Replace recipes with their EVL variants.
 
static void expandVPWidenIntOrFpInduction (VPWidenIntOrFpInductionRecipe *WidenIVR, VPTypeAnalysis &TypeInfo)
 Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.
 
static void expandVPWidenPointerInduction (VPWidenPointerInductionRecipe *R, VPTypeAnalysis &TypeInfo)
 Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.
 
static VPExpressionRecipetryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
 This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
 
static VPExpressionRecipetryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
 This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
 
static void tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
 This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
 
static bool canNarrowLoad (VPWidenRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx)
 Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
 
static bool isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, unsigned VF, VPTypeAnalysis &TypeInfo, unsigned VectorRegWidth)
 Returns true if IR is a full interleave group with factor and number of members both equal to VF.
 
static bool isAlreadyNarrow (VPValue *VPV)
 Returns true if VPValue is a narrow VPValue.
 

Detailed Description

This file implements a set of utility VPlan to VPlan transformations.

Definition in file VPlanTransforms.cpp.

Function Documentation

◆ addReplicateRegions()

static void addReplicateRegions ( VPlan Plan)
static

◆ addVPLaneMaskPhiAndUpdateExitBranch()

static VPActiveLaneMaskPHIRecipe * addVPLaneMaskPhiAndUpdateExitBranch ( VPlan Plan,
bool  DataAndControlFlowWithoutRuntimeCheck 
)
static

◆ canNarrowLoad()

static bool canNarrowLoad ( VPWidenRecipe WideMember0,
unsigned  OpIdx,
VPValue OpV,
unsigned  Idx 
)
static

Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.

V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group.

Definition at line 3561 of file VPlanTransforms.cpp.

References llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), Idx, IR, and OpIdx.

◆ collectUsersRecursively()

static SmallVector< VPUser * > collectUsersRecursively ( VPValue V)
static

◆ createReplicateRegion()

static VPRegionBlock * createReplicateRegion ( VPReplicateRecipe PredRecipe,
VPlan Plan 
)
static

◆ createScalarIVSteps()

static VPScalarIVStepsRecipe * createScalarIVSteps ( VPlan Plan,
InductionDescriptor::InductionKind  Kind,
Instruction::BinaryOps  InductionOpcode,
FPMathOperator FPBinOp,
Instruction TruncI,
VPValue StartV,
VPValue Step,
DebugLoc  DL,
VPBuilder Builder 
)
static

◆ expandVPWidenIntOrFpInduction()

static void expandVPWidenIntOrFpInduction ( VPWidenIntOrFpInductionRecipe WidenIVR,
VPTypeAnalysis TypeInfo 
)
static

Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.

In the following example:

vector.ph: Successor(s): vector loop

<x1> vector loop: { vector.body: WIDEN-INDUCTION i = phi start, step, vf ... EMIT branch-on-count ... No successors }

WIDEN-INDUCTION will get expanded to:

vector.ph: ... vp<induction.start> = ... vp<induction.increment> = ...

Successor(s): vector loop

<x1> vector loop: { vector.body: ir<i> = WIDEN-PHI vp<induction.start>, vp<vec.ind.next> ... vp<vec.ind.next> = add ir<i>, vp<induction.increment> EMIT branch-on-count ... No successors }

Definition at line 2770 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), llvm::VPInstruction::Broadcast, llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createScalarCast(), llvm::VPBuilder::createScalarZExtOrTrunc(), llvm::VPBuilder::createWidenCast(), DL, llvm::IntegerType::get(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPWidenIntOrFpInductionRecipe::getLastUnrolledPartOperand(), llvm::VPRecipeBase::getParent(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPBlockBase::getPlan(), llvm::Type::getScalarSizeInBits(), llvm::VPWidenIntOrFpInductionRecipe::getSplatVFValue(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::InductionDescriptor::IK_IntInduction, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::VPValue::replaceAllUsesWith(), llvm::VPBuilder::setInsertPoint(), and llvm::VPInstruction::StepVector.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

◆ expandVPWidenPointerInduction()

static void expandVPWidenPointerInduction ( VPWidenPointerInductionRecipe R,
VPTypeAnalysis TypeInfo 
)
static

Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.

In the following example:

<x1> vector loop: { vector.body: EMIT ir<ptr.iv> = WIDEN-POINTER-INDUCTION start, step, vf ... EMIT branch-on-count ... }

WIDEN-POINTER-INDUCTION will get expanded to:

<x1> vector loop: { vector.body: EMIT-SCALAR pointer.phi = phi start, ptr.ind EMIT mul = mul stepvector, step EMIT vector.gep = wide-ptradd pointer.phi, mul ... EMIT ptr.ind = ptradd pointer.phi, vf EMIT branch-on-count ... }

Definition at line 2882 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createPtrAdd(), llvm::VPBuilder::createScalarPhi(), llvm::VPBuilder::createScalarZExtOrTrunc(), DL, llvm::VPBlockBase::getExitingBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::InductionDescriptor::IK_PtrInduction, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isPointerTy(), llvm::Offset, llvm::VPBuilder::setInsertPoint(), llvm::VPInstruction::StepVector, and llvm::VPInstruction::WidePtrAdd.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

◆ findHeaderMask()

static VPSingleDefRecipe * findHeaderMask ( VPlan Plan)
static

Collect the header mask with the pattern: (ICMP_ULE, WideCanonicalIV, backedge-taken-count) TODO: Introduce explicit recipe for header-mask instead of searching for the header-mask pattern manually.

Definition at line 2085 of file VPlanTransforms.cpp.

References assert(), llvm::count_if(), llvm::iterator_range< IteratorT >::end(), llvm::find_if(), llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::vputils::isHeaderMask(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::addActiveLaneMask(), and transformRecipestoEVLRecipes().

◆ getOptimizableIVOf()

static VPWidenInductionRecipe * getOptimizableIVOf ( VPValue VPV,
ScalarEvolution SE 
)
static

◆ getPredicatedMask()

VPValue * getPredicatedMask ( VPRegionBlock R)

If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.

Definition at line 217 of file VPlanTransforms.cpp.

Referenced by mergeReplicateRegionsIntoSuccessors().

◆ getPredicatedThenBlock()

static VPBasicBlock * getPredicatedThenBlock ( VPRegionBlock R)
static

If R is a triangle region, return the 'then' block of the triangle.

Definition at line 227 of file VPlanTransforms.cpp.

Referenced by mergeReplicateRegionsIntoSuccessors().

◆ hoistPreviousBeforeFORUsers()

static bool hoistPreviousBeforeFORUsers ( VPFirstOrderRecurrencePHIRecipe FOR,
VPRecipeBase Previous,
VPDominatorTree VPDT 
)
static

◆ isAlreadyNarrow()

static bool isAlreadyNarrow ( VPValue VPV)
static

Returns true if VPValue is a narrow VPValue.

Definition at line 3608 of file VPlanTransforms.cpp.

References llvm::VPValue::isLiveIn().

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

◆ isConditionTrueViaVFAndUF()

static bool isConditionTrueViaVFAndUF ( VPValue Cond,
VPlan Plan,
ElementCount  BestVF,
unsigned  BestUF,
ScalarEvolution SE 
)
static

◆ isConsecutiveInterleaveGroup()

static bool isConsecutiveInterleaveGroup ( VPInterleaveRecipe InterleaveR,
unsigned  VF,
VPTypeAnalysis TypeInfo,
unsigned  VectorRegWidth 
)
static

Returns true if IR is a full interleave group with factor and number of members both equal to VF.

The interleave group must also access the full vector width VectorRegWidth.

Definition at line 3577 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPDef::definedValues(), llvm::VPInterleaveRecipe::getInterleaveGroup(), llvm::Type::getScalarSizeInBits(), llvm::VPInterleaveRecipe::getStoredValues(), llvm::VPDef::getVPValue(), and llvm::VPTypeAnalysis::inferScalarType().

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

◆ isDeadRecipe()

static bool isDeadRecipe ( VPRecipeBase R)
static

Returns true if R is dead and can be removed.

Definition at line 531 of file VPlanTransforms.cpp.

References llvm::all_of(), and llvm::PatternMatch::match().

Referenced by recursivelyDeleteDeadRecipes(), and llvm::VPlanTransforms::removeDeadRecipes().

◆ legalizeAndOptimizeInductions()

static void legalizeAndOptimizeInductions ( VPlan Plan)
static

Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.

If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.

Definition at line 637 of file VPlanTransforms.cpp.

References collectUsersRecursively(), llvm::VPBuilder::createPtrAdd(), createScalarIVSteps(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::InductionDescriptor::IK_IntInduction, llvm::vputils::isSingleScalar(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and Users.

Referenced by llvm::VPlanTransforms::optimize().

◆ licm()

static void licm ( VPlan Plan)
static

◆ mergeBlocksIntoPredecessors()

static bool mergeBlocksIntoPredecessors ( VPlan Plan)
static

◆ mergeReplicateRegionsIntoSuccessors()

static bool mergeReplicateRegionsIntoSuccessors ( VPlan Plan)
static

◆ narrowToSingleScalarRecipes()

static void narrowToSingleScalarRecipes ( VPlan Plan)
static

◆ optimizeEarlyExitInductionUser()

static VPValue * optimizeEarlyExitInductionUser ( VPlan Plan,
VPTypeAnalysis TypeInfo,
VPBlockBase PredVPBB,
VPValue Op,
ScalarEvolution SE 
)
static

◆ optimizeLatchExitInductionUser()

static VPValue * optimizeLatchExitInductionUser ( VPlan Plan,
VPTypeAnalysis TypeInfo,
VPBlockBase PredVPBB,
VPValue Op,
DenseMap< VPValue *, VPValue * > &  EndValues,
ScalarEvolution SE 
)
static

◆ optimizeMaskToEVL()

static VPRecipeBase * optimizeMaskToEVL ( VPValue HeaderMask,
VPRecipeBase CurRecipe,
VPTypeAnalysis TypeInfo,
VPValue AllOneMask,
VPValue EVL 
)
static

Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.

Returns nullptr if no EVL-based recipe could be created. HeaderMask Header Mask. CurRecipe Recipe to be transform. TypeInfo VPlan-based type analysis. AllOneMask The vector mask parameter of vector-predication intrinsics. EVL The explicit vector length parameter of vector-predication intrinsics.

Adjust any end pointers so that they point to the end of EVL lanes not VF.

Definition at line 2170 of file VPlanTransforms.cpp.

References Addr, llvm::all_of(), assert(), llvm::TypeSwitch< T, ResultT >::Case(), llvm::VPVectorEndPointerRecipe::clone(), llvm::Default, llvm::VPWidenMemoryRecipe::getAddr(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenMemoryRecipe::getMask(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertBefore(), LHS, llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), RHS, and llvm::VPUser::setOperand().

Referenced by transformRecipestoEVLRecipes().

◆ optimizeVectorInductionWidthForTCAndVFUF()

static bool optimizeVectorInductionWidthForTCAndVFUF ( VPlan Plan,
ElementCount  BestVF,
unsigned  BestUF 
)
static

◆ recursivelyDeleteDeadRecipes()

static void recursivelyDeleteDeadRecipes ( VPValue V)
static

◆ removeRedundantCanonicalIVs()

static void removeRedundantCanonicalIVs ( VPlan Plan)
static

◆ removeRedundantExpandSCEVRecipes()

static void removeRedundantExpandSCEVRecipes ( VPlan Plan)
static

Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.

Definition at line 904 of file VPlanTransforms.cpp.

References llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::make_early_inc_range(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().

Referenced by llvm::VPlanTransforms::optimize().

◆ removeRedundantInductionCasts()

static void removeRedundantInductionCasts ( VPlan Plan)
static

Remove redundant casts of inductions.

Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.

Definition at line 465 of file VPlanTransforms.cpp.

References llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::optimize().

◆ simplifyBlends()

static void simplifyBlends ( VPlan Plan)
static

◆ simplifyBranchConditionForVFAndUF()

static bool simplifyBranchConditionForVFAndUF ( VPlan Plan,
ElementCount  BestVF,
unsigned  BestUF,
PredicatedScalarEvolution PSE 
)
static

Try to simplify the branch condition of Plan.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 1479 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::VPBasicBlock::appendRecipe(), assert(), B, llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, llvm::CallingConv::C, Cond, llvm::VPBlockUtils::connectBlocks(), llvm::VPBuilder::createNaryOp(), llvm::VPBlockUtils::disconnectBlocks(), llvm::ScalarEvolution::getElementCount(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPDef::getVPSingleValue(), llvm::CmpInst::ICMP_ULE, isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::SCEV::isZero(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy(), llvm::VPValue::replaceAllUsesWith(), llvm::VPlanTransforms::simplifyRecipes(), llvm::VPInstruction::StepVector, and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

◆ simplifyRecipe()

static void simplifyRecipe ( VPRecipeBase R,
VPTypeAnalysis TypeInfo 
)
static

◆ sinkRecurrenceUsersAfterPrevious()

static bool sinkRecurrenceUsersAfterPrevious ( VPFirstOrderRecurrencePHIRecipe FOR,
VPRecipeBase Previous,
VPDominatorTree VPDT 
)
static

Sink users of FOR after the recipe defining the previous value Previous of the recurrence.

Returns
true if all users of FOR could be re-arranged as needed or false if it is not possible.

Definition at line 1585 of file VPlanTransforms.cpp.

References A, assert(), B, llvm::VPDef::getNumDefinedValues(), llvm::VPDef::getVPSingleValue(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPDominatorTree::properlyDominates(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), llvm::sort(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::adjustFixedOrderRecurrences().

◆ sinkScalarOperands()

static bool sinkScalarOperands ( VPlan Plan)
static

◆ transformRecipestoEVLRecipes()

static void transformRecipestoEVLRecipes ( VPlan Plan,
VPValue EVL 
)
static

◆ tryToCreateAbstractReductionRecipe()

static void tryToCreateAbstractReductionRecipe ( VPReductionRecipe Red,
VPCostContext Ctx,
VFRange Range 
)
static

This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.

Definition at line 3250 of file VPlanTransforms.cpp.

References llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().

Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().

◆ tryToFoldLiveIns()

static Value * tryToFoldLiveIns ( const VPRecipeBase R,
unsigned  Opcode,
ArrayRef< VPValue * >  Operands,
const DataLayout DL,
VPTypeAnalysis TypeInfo 
)
static

◆ tryToMatchAndCreateExtendedReduction()

static VPExpressionRecipe * tryToMatchAndCreateExtendedReduction ( VPReductionRecipe Red,
VPCostContext Ctx,
VFRange Range 
)
static

◆ tryToMatchAndCreateMulAccumulateReduction()

static VPExpressionRecipe * tryToMatchAndCreateMulAccumulateReduction ( VPReductionRecipe Red,
VPCostContext Ctx,
VFRange Range 
)
static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))).

Definition at line 3151 of file VPlanTransforms.cpp.

References A, B, CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::TargetTransformInfo::getMulAccReductionCost(), llvm::RecurrenceDescriptor::getOpcode(), if(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::InstructionCost::isValid(), llvm::PatternMatch::m_Mul(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::Mul, Range, llvm::TargetTransformInfo::TCK_RecipThroughput, llvm::toVectorTy(), llvm::VPCostContext::TTI, and llvm::VPCostContext::Types.

Referenced by tryToCreateAbstractReductionRecipe().