13#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
14#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
57 template <
bool EnableVerify =
true,
typename PassTy,
typename... ArgsTy>
67 std::string NumberedPassName;
70 Instance = ++PassCounter[{Fn,
PassName}];
72 NumberedPassName = Instance == 1
90 return std::forward<PassTy>(
Pass)(Plan, std::forward<ArgsTy>(Args)...);
92#define RUN_VPLAN_PASS(PASS, ...) \
93 llvm::VPlanTransforms::runPass(#PASS, PASS, __VA_ARGS__)
94#define RUN_VPLAN_PASS_NO_VERIFY(PASS, ...) \
95 llvm::VPlanTransforms::runPass<false>(#PASS, PASS, __VA_ARGS__)
140 PredicatedScalarEvolution &PSE, LoopVersioning *LVer =
nullptr,
141 function_ref<
const BranchProbabilityInfo &()> GetBPI =
nullptr);
155 VPlan &Plan, PredicatedScalarEvolution &PSE,
Loop &OrigLoop,
156 OptimizationRemarkEmitter *ORE,
const VPDominatorTree &VPDT,
157 const MapVector<PHINode *, InductionDescriptor> &Inductions,
158 const MapVector<PHINode *, RecurrenceDescriptor> &
Reductions,
159 const MapVector<PHINode *, ConditionalInductionDescriptor>
160 &ConditionalInductions,
161 const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
162 const SmallPtrSetImpl<PHINode *> &InLoopReductions,
bool AllowReordering);
171 bool OptForSize,
unsigned SCEVCheckThreshold,
172 OptimizationRemarkEmitter *ORE,
Loop *TheLoop);
186 VPlan &Plan, ElementCount VF,
unsigned UF,
187 ElementCount MinProfitableTripCount,
bool RequiresScalarEpilogue,
189 DebugLoc DL, PredicatedScalarEvolution &PSE, VPBasicBlock *CheckBlock);
195 bool RequiresScalarEpilogue,
Loop *OrigLoop,
202 VPlan &Plan, VPValue *MainVectorTripCount,
bool RequiresScalarEpilogue,
203 ElementCount EpilogueVF,
unsigned MainLoopStep,
unsigned EpilogueLoopStep,
204 ScalarEvolution &SE);
214 VPBasicBlock *CheckBlock,
215 bool AddBranchWeights);
217 bool AddBranchWeights);
223 const RuntimePointerChecking &RtPtrChecking,
224 const Loop *TheLoop, ScalarEvolution &SE,
225 ElementCount VF,
unsigned UF,
226 bool AddBranchWeights);
237 VPBasicBlock *MainCheck);
244 VPlan &Plan,
const TargetLibraryInfo &
TLI, PredicatedScalarEvolution &PSE,
252 OptimizationRemarkEmitter *ORE,
276 VPRecipeBuilder &RecipeBuilder,
277 ScalarEvolution &SE);
283 static void unrollByUF(VPlan &Plan,
unsigned UF);
296 PredicatedScalarEvolution &PSE);
301 PredicatedScalarEvolution &PSE);
322 bool UseActiveLaneMaskForControlFlow);
328 const MapVector<Instruction *, uint64_t> &MinBWs);
333 PredicatedScalarEvolution &PSE);
338 PredicatedScalarEvolution &PSE,
340 const VPDominatorTree &VPDT);
357 const std::optional<unsigned> &MaxEVLSafeElements);
373 const SmallPtrSetImpl<
const InterleaveGroup<Instruction> *>
375 const bool &EpilogueAllowed);
381 PredicatedScalarEvolution &PSE,
Loop &L,
382 VPCostContext &Ctx, VFRange &
Range);
398 PredicatedScalarEvolution &PSE,
400 AssumptionCache *AC);
414 Loop *TheLoop, PredicatedScalarEvolution &PSE,
415 DominatorTree &DT, AssumptionCache *AC,
466 static void cse(VPlan &Plan,
const VPDominatorTree &VPDT);
472 PredicatedScalarEvolution &PSE,
494 const TargetTransformInfo &
TTI);
501 PredicatedScalarEvolution &PSE);
508 VPlan &Plan, VPBasicBlock *VectorPHVPBB,
bool TailByMasking,
509 bool RequiresScalarEpilogue, VPValue *Step,
510 std::optional<uint64_t> MaxRuntimeStep = std::nullopt);
528 VPBasicBlock *AliasCheckVPBB,
530 ScalarEvolution &SE);
538 ScalarEvolution &SE,
bool HasBranchWeights);
548 static DenseMap<const SCEV *, Value *>
expandSCEVs(VPlan &Plan,
549 ScalarEvolution &SE);
563 static std::unique_ptr<VPlan>
610 const TargetTransformInfo &
TTI,
612 ElementCount VF,
unsigned UF);
618 std::optional<unsigned> VScaleForTuning);
633 PredicatedScalarEvolution &PSE,
Loop &L);
645 VPRecipeBuilder &RecipeBuilder,
646 VPCostContext &CostCtx);
658 VPRecipeBuilder &RecipeBuilder,
659 VPCostContext &CostCtx);
667 const TargetTransformInfo &
TTI,
668 PredicatedScalarEvolution &PSE);
const TargetLowering & TLI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define LLVM_ABI_FOR_TEST
static constexpr uint32_t MinItersBypassWeights[]
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
SmallVector< MachineOperand, 4 > Cond
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
const Function * getParent() const
Return the enclosing method, or null if none.
Analysis providing branch probability information.
A struct for saving information about induction variables.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Pass interface - Implemented by all 'passes'.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
The main scalar evolution driver.
Represent a constant reference to a string, i.e.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
BasicBlock * getIRBasicBlock() const
Helper class to create VPRecipies from IR instructions.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
This is an optimization pass for GlobalISel generic memory operations.
DenseMap< Value *, const SCEVUnknown * > SymbolicStrideMap
Maps a pointer to its symbolic (non-constant) stride.
UncountableExitStyle
Different methods of handling early exits.
ArrayRef(const T &OneElt) -> ArrayRef< T >
A range of powers-of-2 vectorization factors with fixed start and adjustable end.