This file implements a set of utility VPlan to VPlan transformations. More...
#include "VPlanTransforms.h"#include "VPRecipeBuilder.h"#include "VPlan.h"#include "VPlanAnalysis.h"#include "VPlanCFG.h"#include "VPlanDominatorTree.h"#include "VPlanHelpers.h"#include "VPlanPatternMatch.h"#include "VPlanUtils.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/PostOrderIterator.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/SmallPtrSet.h"#include "llvm/ADT/TypeSwitch.h"#include "llvm/Analysis/IVDescriptors.h"#include "llvm/Analysis/Loads.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"#include "llvm/Analysis/ScopedNoAliasAA.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/MDBuilder.h"#include "llvm/IR/Metadata.h"#include "llvm/IR/ProfDataUtils.h"#include "llvm/Support/Casting.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/TypeSize.h"#include "llvm/Transforms/Utils/LoopUtils.h"Go to the source code of this file.
Classes | |
| class | SinkStoreInfo |
| Helper for extra no-alias checks via known-safe recipe and SCEV. More... | |
| struct | EarlyExitInfo |
Functions | |
| static VPIRMetadata | getMetadataOf (VPRecipeBase *R) |
Returns the metadata attached to R, or an empty set for a recipe that does not carry any. | |
| static bool | canHoistOrSinkWithNoAliasCheck (const MemoryLocation &MemLoc, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, std::optional< SinkStoreInfo > SinkInfo={}) |
Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB. | |
| static Type * | getLoadStoreValueType (VPReplicateRecipe *R, bool IsLoad) |
| Get the value type of the replicate load or store. | |
| template<unsigned Opcode> | |
| static SmallVector< SmallVector< VPReplicateRecipe *, 4 > > | collectGroupedReplicateMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L, function_ref< bool(VPReplicateRecipe *)> FilterFn) |
Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan. | |
| static bool | sinkScalarOperands (VPlan &Plan) |
| static VPBasicBlock * | getPredicatedThenBlock (VPRegionBlock *R) |
If R is a triangle region, return the 'then' block of the triangle. | |
| static bool | mergeReplicateRegionsIntoSuccessors (VPlan &Plan) |
| static VPRegionBlock * | createReplicateRegion (VPReplicateRecipe *PredRecipe, VPRegionBlock *ParentRegion, VPlan &Plan) |
| static void | addReplicateRegions (VPlan &Plan) |
| static void | removeRedundantInductionCasts (VPlan &Plan) |
| Remove redundant casts of inductions. | |
| static bool | tryToRemoveDeadCycle (VPRecipeBase *R) |
| If R is a phi-like recipe starting a dead cycle of recipes, erase all reachable recipes of the dead cycle and return true. | |
| static void | legalizeAndOptimizeInductions (VPlan &Plan) |
| Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only. | |
| static VPWidenInductionRecipe * | getOptimizableIVOf (VPValue *VPV, PredicatedScalarEvolution &PSE) |
Check if VPV is an untruncated wide induction, either before or after the increment. | |
| static VPValue * | optimizeEarlyExitInductionUser (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE) |
| Attempts to optimize the induction variable exit values for users in the early exit block. | |
| static VPValue * | tryToComputeEndValueForInduction (VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPValue *VectorTC) |
Compute the end value for WideIV, unless it is truncated. | |
| static VPValue * | optimizeLatchExitInductionUser (VPlan &Plan, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, PredicatedScalarEvolution &PSE) |
| Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop. | |
| static VPValue * | optimizeLatchExitIVUserViaSCEV (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE, VPValue *ResumeTC, const Loop *L) |
| static void | removeRedundantExpandSCEVRecipes (VPlan &Plan) |
Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression. | |
| static VPValue * | simplifyLogicalRecipe (VPlan &Plan, VPSingleDefRecipe *Def) |
Try to simplify logical and bitwise recipes in Def. | |
| static void | swapSelectBranchWeights (VPRecipeBase &R, VPlan &Plan) |
Swap the branch weights recorded for R, a select whose two selected operands are being swapped,. | |
| static VPValue * | simplifyRecipe (VPlan &Plan, VPSingleDefRecipe *Def) |
Return an existing value or a live in for VPSingleDefRecipe Def if possible. | |
| static bool | isAvailableAtEndOf (VPValue *V, const VPBasicBlock *VPBB) |
Returns true if V is available at the end of VPBB, i.e. | |
| static VPSingleDefRecipe * | combineRecipe (VPlan &Plan, VPSingleDefRecipe *Def, VPCombineBuilder &Builder) |
Combine Def into a simpler recipe. | |
| static void | reassociateHeaderMask (VPlan &Plan) |
| Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g. | |
| static std::optional< Instruction::BinaryOps > | getUnmaskedDivRemOpcode (Intrinsic::ID ID) |
| static void | narrowToSingleScalarRecipes (VPlan &Plan) |
| static void | removeCommonBlendMask (VPBlendRecipe *Blend) |
Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks. | |
| static void | simplifyBlends (VPlan &Plan) |
| Normalize and simplify VPBlendRecipes. | |
| static bool | optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF) |
Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF. | |
| static bool | isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE) |
Return true if Cond is known to be true for given BestVF and BestUF. | |
| static bool | replaceMaskWithCompareForScalarPlan (VPlan &Plan, ElementCount BestVF) |
| static bool | simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE) |
Try to simplify the branch condition of Plan. | |
| static bool | cannotHoistOrSinkRecipe (VPRecipeBase &R, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, bool Sinking=false) |
Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region. | |
| static void | licm (VPlan &Plan) |
Move loop-invariant recipes out of the vector loop region in Plan. | |
| static auto | m_Uncountable (VPValue *&Cond) |
| Matches an exit condition formed by comparing a value loaded from memory with a loop-invariant term. | |
| static auto | m_Countable (VPValue *&Cmp, PredicatedScalarEvolution &PSE, Loop *L) |
| Matches an exit condition formed by comparing the current value of a affine add recurrence in the given loop with a stride of 1 against a loop-invariant term. | |
| static VPValue * | getRecipesForUncountableExit (SmallVectorImpl< VPInstruction * > &Recipes, VPBasicBlock *LatchVPBB) |
| Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison. | |
| static bool | handleUncountableExitsWithSideEffects (VPlan &Plan, SmallVectorImpl< EarlyExitInfo > &Exits, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) |
Update Plan to mask memory operations in the loop based on whether the early exit is taken or not. | |
| static VPExpressionRecipe * | tryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid. | |
| static VPExpressionRecipe * | tryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid. | |
| static void | tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
| This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation. | |
| static VPIRMetadata | getCommonMetadata (ArrayRef< VPReplicateRecipe * > Recipes) |
| template<unsigned Opcode> | |
| static SmallVector< SmallVector< VPReplicateRecipe *, 4 > > | collectComplementaryPredicatedMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) |
| template<typename InstType> | |
| static VPReplicateRecipe * | findRecipeWithMinAlign (ArrayRef< VPReplicateRecipe * > Group) |
| static bool | canSinkStoreWithNoAliasCheck (ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution &PSE, const Loop &L) |
| static bool | canNarrowLoad (VPSingleDefRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx, bool IsScalable) |
Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe. | |
| static bool | canNarrowOps (ArrayRef< VPValue * > Ops, bool IsScalable, DenseMap< VPValue *, SmallVector< VPValue * > > &FirstMembersOf) |
Returns true if the wide recipes in Ops and their operand trees can be narrowed. | |
| static std::optional< ElementCount > | isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI) |
Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF. | |
| static bool | isAlreadyNarrow (VPValue *VPV) |
Returns true if VPValue is a narrow VPValue. | |
| static VPValue * | narrowInterleaveGroupOp (ArrayRef< VPValue * > Members, SmallPtrSetImpl< VPValue * > &NarrowedOps, VPBasicBlock *Preheader) |
| static VPWidenIntOrFpInductionRecipe * | getExpressionIV (VPValue *V) |
Check if V is a binary expression of a widened IV and a loop-invariant value. | |
| static VPValue * | cloneBinOpForScalarIV (VPWidenRecipe *BinOp, VPValue *ScalarIV, VPWidenIntOrFpInductionRecipe *WidenIV) |
Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe. | |
| static std::optional< bool > | getStepDirection (const SCEV *S, ScalarEvolution &SE) |
If S is an affine AddRec, returns true if its step is known to be positive and false if it is known to be negative. | |
| static void | updatePartialReductionPhiAndResult (VPlan &Plan, VPReductionPHIRecipe *Phi, unsigned Factor, RecurKind RK) |
| static bool | areVFParamsOk (const VFInfo &Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution &PSE, const Loop *L) |
Returns true if Info's parameter kinds are compatible with Args. | |
| static Function * | findVectorVariant (CallInst *CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution &PSE, const Loop *L) |
Find a vector variant of CI for VF, respecting MaskRequired. | |
| static CallWideningDecision | decideCallWidening (VPInstruction &VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext &CostCtx) |
Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant. | |
Variables | |
| static cl::opt< bool > | UsePartialReductionsByDefault ("use-partial-reductions-by-default", cl::init(false), cl::Hidden, cl::desc("Use partial reduction intrinsics for " "all supported unordered reductions.")) |
Detailed Description
This file implements a set of utility VPlan to VPlan transformations.
Definition in file VPlanTransforms.cpp.
Function Documentation
◆ addReplicateRegions()
|
static |
Definition at line 597 of file VPlanTransforms.cpp.
References llvm::VPBlockUtils::blocksOnly(), createReplicateRegion(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getExiting(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::RegionBase< Tr >::getParent(), llvm::VPBlockBase::getParent(), llvm::Value::hasName(), llvm::VPBlockUtils::insertOnEdge(), llvm::make_isa_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPRegionBlock::setExiting(), llvm::VPBasicBlock::splitAt(), llvm::SplitBlock(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ areVFParamsOk()
|
static |
Returns true if Info's parameter kinds are compatible with Args.
Definition at line 6017 of file VPlanTransforms.cpp.
References llvm::all_of(), and llvm::PredicatedScalarEvolution::getSE().
Referenced by findVectorVariant().
◆ canHoistOrSinkWithNoAliasCheck()
|
static |
Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.
If SinkInfo is std::nullopt, only recipes that may write to memory are checked (for load hoisting). Otherwise recipes that both read and write memory are checked, and SCEV is used to prove no-alias between the group leader and other replicate recipes (for store sinking).
Definition at line 253 of file VPlanTransforms.cpp.
Referenced by cannotHoistOrSinkRecipe(), canSinkStoreWithNoAliasCheck(), and llvm::VPlanTransforms::hoistPredicatedLoads().
◆ canNarrowLoad()
|
static |
Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group. A live-in or recipe defined outside the loop region can be converted, if it is the same across all lanes, or we can create a BuildVector for it.
Definition at line 4279 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), llvm::VPValue::getScalarType(), IR, and llvm::VPValue::isDefinedOutsideLoopRegions().
Referenced by canNarrowOps().
◆ canNarrowOps()
|
static |
Returns true if the wide recipes in Ops and their operand trees can be narrowed.
FirstMembersOf maps each in-loop value to the first member list it was seen in, across all store groups.
Definition at line 4306 of file VPlanTransforms.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), canNarrowLoad(), canNarrowOps(), llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), llvm::equal(), llvm::vputils::getOpcode(), llvm::isa(), P, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by canNarrowOps(), and llvm::VPlanTransforms::narrowInterleaveGroups().
◆ cannotHoistOrSinkRecipe()
|
static |
Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.
When sinking, passing Sinking = true ensures that assumes aren't sunk.
Definition at line 2430 of file VPlanTransforms.cpp.
References canHoistOrSinkWithNoAliasCheck(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), llvm::vputils::getMemoryLocation(), llvm::isa(), llvm::PatternMatch::m_Intrinsic(), and llvm::SCEVPatternMatch::match().
Referenced by licm().
◆ canSinkStoreWithNoAliasCheck()
|
static |
Definition at line 4139 of file VPlanTransforms.cpp.
References llvm::ArrayRef< T >::back(), canHoistOrSinkWithNoAliasCheck(), llvm::ArrayRef< T >::front(), and llvm::vputils::getMemoryLocation().
Referenced by llvm::VPlanTransforms::sinkPredicatedStores().
◆ cloneBinOpForScalarIV()
|
static |
Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.
Definition at line 4798 of file VPlanTransforms.cpp.
References assert(), llvm::VPWidenRecipe::clone(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPWidenRecipe::getOpcode(), llvm::VPRecipeBase::insertAfter(), and llvm::Instruction::isBinaryOp().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ collectComplementaryPredicatedMemOps()
|
static |
Definition at line 4024 of file VPlanTransforms.cpp.
References llvm::all_equal(), assert(), llvm::bind_back(), collectGroupedReplicateMemOps(), Groups, llvm::VPReplicateRecipe::isPredicated(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::map_range(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ collectGroupedReplicateMemOps()
|
static |
Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.
Definition at line 291 of file VPlanTransforms.cpp.
References A(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::VPRegionBlock::getEntry(), getLoadStoreValueType(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getVectorLoopRegion(), Groups, llvm::isa(), llvm::make_isa_range(), llvm::VPDominatorTree::properlyDominates(), llvm::stable_sort(), llvm::to_vector(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::values(), and llvm::vp_depth_first_deep().
Referenced by collectComplementaryPredicatedMemOps().
◆ combineRecipe()
|
static |
Combine Def into a simpler recipe.
May modify or create new recipes via Builder.
Definition at line 1407 of file VPlanTransforms.cpp.
References llvm::all_equal(), llvm::all_of(), llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::Broadcast, C(), llvm::cast(), llvm::dyn_cast(), llvm::dyn_cast_if_present(), llvm::APInt::exactLogBase2(), llvm::CmpInst::FCMP_UNO, llvm::VPlanPatternMatch::findUserOf(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::CmpInst::getInversePredicate(), llvm::VPUser::getOperand(), llvm::Type::getScalarSizeInBits(), llvm::VPlan::getTrue(), llvm::VPDef::getVPSingleValue(), llvm::VPlan::getZero(), llvm::isa(), isAvailableAtEndOf(), llvm::APInt::isPowerOf2(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_Add(), llvm::MIPatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_Mul(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_False(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_LShr(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SelectLike(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificCmp(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_True(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_UDiv(), llvm::PatternMatch::m_URem(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPPhi(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), simplifyRecipe(), llvm::SmallVectorTemplateCommon< T, typename >::size(), swapSelectBranchWeights(), llvm::to_vector(), X, and Y.
Referenced by llvm::VPlanTransforms::combineRecipes().
◆ createReplicateRegion()
|
static |
Definition at line 550 of file VPlanTransforms.cpp.
References assert(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createReplicateRegion(), llvm::VPlan::createVPBasicBlock(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPReplicateRecipe::getMask(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::DebugLoc::getUnknown(), llvm::VPBlockUtils::insertTwoBlocksAfter(), llvm::VPReplicateRecipe::isSingleScalar(), llvm::VPReplicateRecipe::operandsWithoutMask(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::user_empty().
Referenced by addReplicateRegions().
◆ decideCallWidening()
|
static |
Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.
Definition at line 6079 of file VPlanTransforms.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::VPReplicateRecipe::computeCallCost(), llvm::VPWidenCallRecipe::computeCallCost(), llvm::VPWidenIntrinsicRecipe::computeCallCost(), findVectorVariant(), llvm::InstructionCost::getInvalid(), llvm::VPValue::getLiveInIRValue(), llvm::VPInstruction::getNumOperandsWithoutMask(), llvm::VPUser::getOperand(), llvm::VPRecipeValue::getScalarType(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::getVectorIntrinsicIDForCall(), IntrinsicCost, llvm::VPCostContext::isFreeScalarIntrinsic(), llvm::VPCostContext::isMaskRequired(), llvm::ElementCount::isScalar(), llvm::VPCostContext::L, llvm::VPCostContext::PSE, llvm::VPCostContext::TLI, and llvm::VPCostContext::willBeScalarized().
Referenced by llvm::VPlanTransforms::makeCallWideningDecisions().
◆ findRecipeWithMinAlign()
|
static |
Definition at line 4082 of file VPlanTransforms.cpp.
References A(), B(), llvm::cast(), and llvm::min_element().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ findVectorVariant()
|
static |
Find a vector variant of CI for VF, respecting MaskRequired.
Returns the variant function, or nullptr. Masked variants are assumed to take the mask as a trailing parameter.
Definition at line 6044 of file VPlanTransforms.cpp.
References areVFParamsOk(), llvm::find_if(), llvm::Module::getFunction(), llvm::VFDatabase::getMappings(), llvm::Instruction::getModule(), and llvm::CallBase::isNoBuiltin().
Referenced by decideCallWidening().
◆ getCommonMetadata()
|
static |
Definition at line 4012 of file VPlanTransforms.cpp.
References llvm::VPIRMetadata::clearExecutionFrequency(), llvm::drop_begin(), llvm::ArrayRef< T >::front(), and llvm::VPIRMetadata::intersect().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ getExpressionIV()
|
static |
Check if V is a binary expression of a widened IV and a loop-invariant value.
Returns the widened IV if found, nullptr otherwise.
Definition at line 4779 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::VPValue::isDefinedOutsideLoopRegions(), llvm::Instruction::isIntDivRem(), and std::swap().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ getLoadStoreValueType()
|
static |
Get the value type of the replicate load or store.
IsLoad indicates whether it is a load.
Definition at line 282 of file VPlanTransforms.cpp.
References llvm::Type::getScalarType().
Referenced by collectGroupedReplicateMemOps().
◆ getMetadataOf()
|
static |
Returns the metadata attached to R, or an empty set for a recipe that does not carry any.
Definition at line 52 of file VPlanTransforms.cpp.
References llvm::dyn_cast().
Referenced by legalizeAndOptimizeInductions(), and narrowToSingleScalarRecipes().
◆ getOptimizableIVOf()
|
static |
Check if VPV is an untruncated wide induction, either before or after the increment.
If so return the header IV (before the increment), otherwise return null.
Definition at line 876 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm_unreachable, llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_FAdd(), llvm::VPlanPatternMatch::m_GetElementPtr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::narrowInductionTruncates(), optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().
◆ getPredicatedThenBlock()
|
static |
If R is a triangle region, return the 'then' block of the triangle.
Definition at line 420 of file VPlanTransforms.cpp.
References llvm::cast(), and llvm::dyn_cast().
Referenced by mergeReplicateRegionsIntoSuccessors().
◆ getRecipesForUncountableExit()
|
static |
Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison.
The recipes are stored in Recipes.
Definition at line 3166 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPBasicBlock::getTerminator(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Freeze(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::none_of(), llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by handleUncountableExitsWithSideEffects().
◆ getStepDirection()
|
static |
If S is an affine AddRec, returns true if its step is known to be positive and false if it is known to be negative.
Returns std::nullopt if S is not an affine AddRec, or if the sign of its step cannot be determined.
Definition at line 4817 of file VPlanTransforms.cpp.
References llvm::ScalarEvolution::isKnownNegative(), llvm::ScalarEvolution::isKnownPositive(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_AffineAddRec(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ getUnmaskedDivRemOpcode()
|
static |
Definition at line 1820 of file VPlanTransforms.cpp.
Referenced by narrowToSingleScalarRecipes().
◆ handleUncountableExitsWithSideEffects()
|
static |
Update Plan to mask memory operations in the loop based on whether the early exit is taken or not.
We're currently expecting to find a loop with properties similar to the following:
for.body: ir<indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0> EMIT ir<arrayidx> = getelementptr inbounds nuw ir< >, ir<indvars.iv> EMIT-SCALAR ir<%0> = load ir<arrayidx> EMIT ir<cmp1> = icmp sgt ir<%0>, ir<5> EMIT vp<%1> = masked-cond ir<cmp1> Successor(s): if.end
if.end: EMIT ir<arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<indvars.iv> EMIT-SCALAR ir<%2> = load ir<arrayidx3> EMIT ir<add> = add nsw ir<%2>, ir<42> EMIT ir<arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<indvars.iv> EMIT store ir<add>, ir<arrayidx5> EMIT ir<indvars.iv.next> = add nuw nsw ir<indvars.iv>, ir<1> EMIT vp<freeze> = freeze ir<%1> EMIT vp<%3> = any-of ir<freeze> EMIT ir<exitcond.not> = icmp eq ir<indvars.iv.next>, ir<10000> EMIT branch-on-two-conds vp<%3>, ir<exitcond.not> Successor(s): middle.block, middle.block, for.body
We currently expect LoopVectorizationLegality to ensure that:
- There must also be a counted exit. We will need to support speculative or first-faulting loads before we can remove this restriction.
- Any stores within the loop must not alias with the load used for the uncountable exit. We can relax this a bit with runtime aliasing checks.
- Other memory operations in the loop can take place before or after the uncountable exit, but must also be unconditional. We need to support combining the conditions in VPlanPredicator.
- The loop must have a single unconditional load contributing to the uncountable exit comparison, and the other term must be loop-invariant. Improving upon this requires work in getRecipesForUncountableExit to handle more complex recipe graphs.
Definition at line 3322 of file VPlanTransforms.cpp.
References _, llvm::VPInstruction::ActiveLaneMask, assert(), llvm::cast(), Cond, llvm::VPBuilderBase< InserterTy >::createAdd(), llvm::VPBuilderBase< InserterTy >::createFirstActiveLane(), llvm::VPBuilderBase< InserterTy >::createICmp(), llvm::VPBuilderBase< InserterTy >::createNaryOp(), llvm::VPBuilderBase< InserterTy >::createScalarZExtOrTrunc(), llvm::dwarf_linker::DebugLoc, llvm::VPBlockUtils::disconnectBlocks(), DL, llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::VPBasicBlock::end(), llvm::erase(), llvm::VPInstruction::ExtractLane, llvm::find_singleton(), llvm::VPBasicBlock::front(), llvm::ScalarEvolution::getConstant(), llvm::VPlan::getDataLayout(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPUser::getOperand(), getRecipesForUncountableExit(), llvm::VPlan::getScalarPreheader(), llvm::VPRecipeValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::VPlan::getZero(), I, llvm::CmpInst::ICMP_EQ, llvm::is_contained(), llvm::isDereferenceableAndAlignedInLoop(), IV, llvm::Load, llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::range_size(), llvm::LoopVectorizationUtils::reportVectorizationFailure(), llvm::reverse(), llvm::VPUser::setOperand(), and llvm::vp_rpo_plain_cfg_loop_body().
Referenced by llvm::VPlanTransforms::handleUncountableEarlyExits().
◆ isAlreadyNarrow()
Returns true if VPValue is a narrow VPValue.
Definition at line 4402 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), and llvm::isa().
Referenced by narrowInterleaveGroupOp(), and llvm::VPlanTransforms::narrowInterleaveGroups().
◆ isAvailableAtEndOf()
|
static |
Returns true if V is available at the end of VPBB, i.e.
it either is a live-in from the original IR or defined in VPBB.
Definition at line 1383 of file VPlanTransforms.cpp.
References llvm::VPRecipeBase::getParent(), and llvm::isa().
Referenced by combineRecipe().
◆ isConditionTrueViaVFAndUF()
|
static |
Return true if Cond is known to be true for given BestVF and BestUF.
Definition at line 2105 of file VPlanTransforms.cpp.
References llvm::any_of(), assert(), C(), Cond, llvm::VPRegionBlock::getCanonicalIV(), llvm::ScalarEvolution::getElementCount(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::isa(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by simplifyBranchConditionForVFAndUF().
◆ isConsecutiveInterleaveGroup()
|
static |
Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.
The interleave group must also access the full vector width.
Definition at line 4358 of file VPlanTransforms.cpp.
References llvm::all_of(), assert(), llvm::VPDef::definedValues(), llvm::VPInterleaveBase::getInterleaveGroup(), llvm::VPInterleaveBase::getMask(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getScalarType(), llvm::VPInterleaveBase::getStoredValues(), llvm::VPDef::getVPValue(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::TargetTransformInfo::RGK_ScalableVector, and Size.
Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().
◆ legalizeAndOptimizeInductions()
|
static |
Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.
Definition at line 777 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), llvm::vputils::collectUsersRecursively(), llvm::vputils::createScalarIVSteps(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), getMetadataOf(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::APInt::isNonNegative(), llvm::vputils::isSingleScalar(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Binary(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_isa_range(), llvm::PatternMatch::match(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reverse(), llvm::vputils::scalarizeVPWidenPointerInduction(), Users, and llvm::VPIRFlags::WrapFlagsTy::withoutNoSignedWrap().
Referenced by llvm::VPlanTransforms::optimize().
◆ licm()
|
static |
Move loop-invariant recipes out of the vector loop region in Plan.
Definition at line 2451 of file VPlanTransforms.cpp.
References llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), cannotHoistOrSinkRecipe(), llvm::cast(), llvm::dyn_cast(), llvm::VPBasicBlock::end(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::make_early_inc_range(), llvm::VPDominatorTree::properlyDominates(), llvm::reverse(), and llvm::vp_depth_first_shallow().
◆ m_Countable()
|
static |
Matches an exit condition formed by comparing the current value of a affine add recurrence in the given loop with a stride of 1 against a loop-invariant term.
Binds the comparison for the condition.
Definition at line 3080 of file VPlanTransforms.cpp.
Referenced by llvm::VPlanTransforms::splitCombinedExits().
◆ m_Uncountable()
|
static |
Matches an exit condition formed by comparing a value loaded from memory with a loop-invariant term.
Binds the comparison for the condition.
Definition at line 3046 of file VPlanTransforms.cpp.
References Cond, llvm::PatternMatch::m_c_Cmp(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::VPlanPatternMatch::m_VPInstruction(), and llvm::VPlanPatternMatch::m_VPValue().
Referenced by llvm::VPlanTransforms::splitCombinedExits().
◆ mergeReplicateRegionsIntoSuccessors()
Definition at line 442 of file VPlanTransforms.cpp.
References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPIRMetadata::clearExecutionFrequency(), llvm::VPBlockUtils::connectBlocks(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallPtrSetImplBase::empty(), llvm::VPlan::getContext(), llvm::VPlan::getEntry(), llvm::VPIRMetadata::getExecutionFrequency(), llvm::VPBasicBlock::getFirstNonPhi(), getPredicatedThenBlock(), llvm::VPBlockBase::getSingleSuccessor(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), llvm::VPIRMetadata::setExecutionFrequency(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ narrowInterleaveGroupOp()
|
static |
Definition at line 4413 of file VPlanTransforms.cpp.
References llvm::all_of(), llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPInstruction::BuildVector, llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBuilderBase< InserterTy >::createWidenLoad(), llvm::ArrayRef< T >::drop_front(), llvm::dyn_cast(), E(), llvm::ArrayRef< T >::front(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), isAlreadyNarrow(), N, narrowInterleaveGroupOp(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by narrowInterleaveGroupOp().
◆ narrowToSingleScalarRecipes()
|
static |
Definition at line 1835 of file VPlanTransforms.cpp.
References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastLane, llvm::VPInstruction::ExtractLastPart, llvm::VPInstruction::ExtractPenultimateElement, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPlan::getEntry(), getMetadataOf(), llvm::vputils::getOpcode(), llvm::DebugLoc::getUnknown(), getUnmaskedDivRemOpcode(), llvm::VPlan::hasScalarVFOnly(), llvm::is_contained(), llvm::isa(), llvm::vputils::isDeadRecipe(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::make_early_inc_range(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), Opc, llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::optimize().
◆ optimizeEarlyExitInductionUser()
|
static |
Attempts to optimize the induction variable exit values for users in the early exit block.
Definition at line 933 of file VPlanTransforms.cpp.
References B(), llvm::cast(), DL, llvm::dyn_cast_or_null(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), getOptimizableIVOf(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_FirstActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeLatchExitInductionUser()
|
static |
Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
Definition at line 1013 of file VPlanTransforms.cpp.
References assert(), B(), llvm::cast(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), getOptimizableIVOf(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getZero(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm_unreachable, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeLatchExitIVUserViaSCEV()
|
static |
Definition at line 1060 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), DL, llvm::VPSCEVExpander::expand(), llvm::VPlan::getConstantInt(), llvm::VPValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::DebugLoc::getUnknown(), llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::Type::isIntOrPtrTy(), llvm::Type::isPointerTy(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_AffineAddRec(), llvm::SCEVPatternMatch::m_SpecificLoop(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeVectorInductionWidthForTCAndVFUF()
|
static |
Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
Definition at line 2031 of file VPlanTransforms.cpp.
References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::cast(), llvm::IntegerType::get(), llvm::APInt::getActiveBits(), llvm::VPlan::getBackedgeTakenCount(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getSingleUser(), llvm::VPlan::getTripCount(), llvm::VPWidenIntOrFpInductionRecipe::getTruncInst(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::VPlan::getZero(), llvm::VPValue::hasMoreThanOneUniqueUser(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::PowerOf2Ceil(), llvm::APIntOps::RoundingUDiv(), uint64_t, and llvm::APInt::UP.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ reassociateHeaderMask()
|
static |
Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.
in optimizeEVLMasks.
Definition at line 1795 of file VPlanTransforms.cpp.
References llvm::append_range(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPRegionBlock::getHeaderMask(), llvm::VPlan::getVectorLoopRegion(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::VPValue::users(), users, X, and Y.
Referenced by llvm::VPlanTransforms::optimize().
◆ removeCommonBlendMask()
|
static |
Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.
Definition at line 1936 of file VPlanTransforms.cpp.
References llvm::VPValue::getDefiningRecipe(), llvm::VPBlendRecipe::getMask(), llvm::VPBlendRecipe::getNumIncomingValues(), llvm::VPUser::getOperand(), I, llvm::VPBlendRecipe::isNormalized(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPBlendRecipe::setMask().
Referenced by simplifyBlends().
◆ removeRedundantExpandSCEVRecipes()
|
static |
Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.
Definition at line 1150 of file VPlanTransforms.cpp.
References llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getTripCount(), llvm::make_early_inc_range(), llvm::make_isa_range(), llvm::VPlan::resetTripCount(), and llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().
Referenced by llvm::VPlanTransforms::optimize().
◆ removeRedundantInductionCasts()
|
static |
Remove redundant casts of inductions.
Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.
Definition at line 674 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::make_isa_range(), llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().
Referenced by llvm::VPlanTransforms::optimize().
◆ replaceMaskWithCompareForScalarPlan()
|
static |
Definition at line 2140 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::VPlan::getConstantInt(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::CmpInst::ICMP_ULT, llvm::ElementCount::isScalar(), llvm::PatternMatch::m_ConstantInt(), llvm::VPlanPatternMatch::m_ExtractVectorForPart(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_WideActiveLaneMask(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), and uint64_t.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ simplifyBlends()
|
static |
Normalize and simplify VPBlendRecipes.
Should be run after combineRecipes to make sure the masks are simplified.
Simplify BLEND a, b, Not(mask) -> BLEND b, a, mask.
Definition at line 1953 of file VPlanTransforms.cpp.
References llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::cast_or_null(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPlanPatternMatch::m_False(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::make_isa_range(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::vputils::recursivelyDeleteDeadRecipes(), removeCommonBlendMask(), llvm::VPValue::replaceAllUsesWith(), llvm::SmallPtrSetImplBase::size(), llvm::VPValue::user_empty(), and llvm::vp_depth_first_shallow().
Referenced by llvm::VPlanTransforms::optimize().
◆ simplifyBranchConditionForVFAndUF()
|
static |
Try to simplify the branch condition of Plan.
This may restrict the resulting plan to BestVF and BestUF.
Definition at line 2180 of file VPlanTransforms.cpp.
References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, C(), Cond, llvm::ScalarEvolution::getElementCount(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_ULE, llvm::isa(), isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::PatternMatch::m_Add(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractVectorForPart(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_WideActiveLaneMask(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), and llvm::Offset.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ simplifyLogicalRecipe()
|
static |
Try to simplify logical and bitwise recipes in Def.
Definition at line 1169 of file VPlanTransforms.cpp.
References assert(), llvm::VPlan::getAllOnesValue(), llvm::VPlan::getFalse(), llvm::VPlan::getZero(), llvm::CmpInst::ICMP_NE, llvm::MIPatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_c_BinaryAnd(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_LogicalAnd(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_False(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), and X.
Referenced by simplifyRecipe().
◆ simplifyRecipe()
|
static |
Return an existing value or a live in for VPSingleDefRecipe Def if possible.
This shouldn't create or modify recipes.
Definition at line 1238 of file VPlanTransforms.cpp.
References A(), llvm::all_equal(), B(), llvm::cast(), DL, llvm::drop_begin(), llvm::dyn_cast(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPlan::getConcreteUF(), llvm::VPlan::getDataLayout(), llvm::VPRecipeBase::getRegion(), llvm::VPlan::getZero(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::vputils::isSingleScalar(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_AShr(), llvm::PatternMatch::m_BitCast(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_Mul(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ComputeReductionResult(), llvm::PatternMatch::m_ConstantInt(), llvm::VPlanPatternMatch::m_DerivedIV(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLane(), llvm::VPlanPatternMatch::m_ExtractLastPart(), llvm::VPlanPatternMatch::m_ExtractPenultimateElement(), llvm::PatternMatch::m_LShr(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Trunc(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPIRValue(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::vputils::onlyFirstLaneUsed(), simplifyLogicalRecipe(), llvm::vputils::tryToFoldLiveIns(), and uint64_t.
Referenced by combineRecipe().
◆ sinkScalarOperands()
Definition at line 325 of file VPlanTransforms.cpp.
References llvm::all_of(), llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), Changed, llvm::VPSingleDefRecipe::clone(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::vputils::doesGeneratePerAllLanes(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPRecipeValue::getScalarType(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::VPlan::hasScalarVFOnly(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::VPRecipeBase::insertBefore(), llvm::isa_and_nonnull(), llvm::vputils::isSingleScalar(), llvm::make_filter_range(), llvm::VPRecipeBase::moveBefore(), llvm::VPUser::operands(), llvm::VPValue::replaceUsesWithIf(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::VPValue::users(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ swapSelectBranchWeights()
|
static |
Swap the branch weights recorded for R, a select whose two selected operands are being swapped,.
Definition at line 1223 of file VPlanTransforms.cpp.
References assert(), llvm::MDBuilder::createBranchWeights(), llvm::dyn_cast(), llvm::extractBranchWeights(), llvm::VPlan::getContext(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by combineRecipe().
◆ tryToComputeEndValueForInduction()
|
static |
Compute the end value for WideIV, unless it is truncated.
Creates a VPDerivedIVRecipe for non-canonical inductions.
Definition at line 979 of file VPlanTransforms.cpp.
References llvm::VPBuilderBase< InserterTy >::createDerivedIV(), llvm::VPBuilderBase< InserterTy >::createScalarCast(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ tryToCreateAbstractReductionRecipe()
|
static |
This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
Definition at line 3977 of file VPlanTransforms.cpp.
References llvm::VPBlockBase::getParent(), llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().
Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().
◆ tryToMatchAndCreateExtendedReduction()
|
static |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
The created recipe must be decomposed to its constituent recipes before execution.
Definition at line 3755 of file VPlanTransforms.cpp.
References A(), assert(), llvm::cast(), CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::InstructionCost::getInvalid(), llvm::RecurrenceDescriptor::getOpcode(), llvm::Type::getScalarType(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatchHelpers::m_Isa(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Range, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().
Referenced by tryToCreateAbstractReductionRecipe().
◆ tryToMatchAndCreateMulAccumulateReduction()
|
static |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))). reduce.fadd(fmul(ext(A), ext(B)))
Definition at line 3808 of file VPlanTransforms.cpp.
References A(), B(), llvm::canConstantBeExtended(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::RecurrenceDescriptor::getOpcode(), llvm::VPWidenCastRecipe::getOpcode(), llvm::VPUser::getOperand(), llvm::TargetTransformInfo::getPartialReductionExtendKind(), llvm::Type::getScalarType(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Mul, Range, llvm::Sub, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().
Referenced by tryToCreateAbstractReductionRecipe().
◆ tryToRemoveDeadCycle()
|
static |
If R is a phi-like recipe starting a dead cycle of recipes, erase all reachable recipes of the dead cycle and return true.
Otherwise leave the plan unchanged and return false.
Definition at line 713 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::vputils::collectUsersRecursively(), llvm::dyn_cast(), llvm::isa(), and llvm::vputils::recursivelyDeleteDeadRecipes().
Referenced by llvm::VPlanTransforms::removeDeadRecipes().
◆ updatePartialReductionPhiAndResult()
|
static |
Definition at line 5582 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), llvm::vputils::findComputeReductionResult(), llvm::FSub, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBuilderBase<>::getToInsertAfter(), llvm::VPInstruction::ReductionStartVector, llvm::VPValue::replaceUsesWithIf(), and llvm::Sub.
Referenced by llvm::VPlanTransforms::createPartialReductions().
Variable Documentation
◆ UsePartialReductionsByDefault
|
static |
Referenced by llvm::VPlanTransforms::createPartialReductions().
Generated on for LLVM by
1.14.0