13#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
14#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
43#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
53 template <
bool EnableVerify =
true,
typename PassTy,
typename... ArgsTy>
57#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
66 <<
"VPlan for loop in '"
72 dbgs() << Plan <<
'\n';
81 return std::forward<PassTy>(
Pass)(Plan, std::forward<ArgsTy>(Args)...);
83#define RUN_VPLAN_PASS(PASS, ...) \
84 llvm::VPlanTransforms::runPass(#PASS, PASS, __VA_ARGS__)
85#define RUN_VPLAN_PASS_NO_VERIFY(PASS, ...) \
86 llvm::VPlanTransforms::runPass<false>(#PASS, PASS, __VA_ARGS__)
132 PredicatedScalarEvolution &PSE, LoopVersioning *LVer =
nullptr);
139 VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop,
140 const MapVector<PHINode *, InductionDescriptor> &Inductions,
141 const MapVector<PHINode *, RecurrenceDescriptor> &
Reductions,
142 const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
143 const SmallPtrSetImpl<PHINode *> &InLoopReductions,
bool AllowReordering);
149 VPlan &Plan,
const DenseSet<BasicBlock *> &BlocksNeedingPredication,
158 PredicatedScalarEvolution &PSE, DominatorTree &DT,
159 AssumptionCache *AC);
167 VPlan &Plan, ElementCount VF,
unsigned UF,
168 ElementCount MinProfitableTripCount,
bool RequiresScalarEpilogue,
175 VPlan &Plan, Value *TripCount, Value *VectorTripCount,
176 bool RequiresScalarEpilogue, ElementCount EpilogueVF,
unsigned EpilogueUF,
177 unsigned MainLoopStep,
unsigned EpilogueLoopStep, ScalarEvolution &SE);
187 bool AddBranchWeights);
194 const TargetLibraryInfo &TLI);
201 OptimizationRemarkEmitter *ORE,
231 static void unrollByUF(VPlan &Plan,
unsigned UF);
243 PredicatedScalarEvolution &PSE);
261 bool UseActiveLaneMaskForControlFlow);
267 const MapVector<Instruction *, uint64_t> &MinBWs);
273 const DenseMap<Value *, const SCEV *> &StridesMap);
289 const std::function<
bool(BasicBlock *)> &BlockNeedsPredication);
298 const std::optional<unsigned> &MaxEVLSafeElements);
314 const SmallPtrSetImpl<
const InterleaveGroup<Instruction> *>
316 VPRecipeBuilder &RecipeBuilder,
const bool &ScalarEpilogueAllowed);
326 VPBasicBlock *LatchVPBB,
327 VPBasicBlock *MiddleVPBB,
370 static void cse(VPlan &Plan);
376 PredicatedScalarEvolution &PSE,
405 PredicatedScalarEvolution &PSE);
410 VPBasicBlock *VectorPHVPBB,
412 bool RequiresScalarEpilogue,
418 VPBasicBlock *VectorPH);
434 static DenseMap<const SCEV *, Value *>
expandSCEVs(VPlan &Plan,
435 ScalarEvolution &SE);
449 static std::unique_ptr<VPlan>
495 std::optional<unsigned> VScaleForTuning);
508 PredicatedScalarEvolution &PSE, Loop &L);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define LLVM_ABI_FOR_TEST
static constexpr uint32_t MinItersBypassWeights[]
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const SmallVectorImpl< MachineOperand > & Cond
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
const Function * getParent() const
Return the enclosing method, or null if none.
A struct for saving information about induction variables.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Pass interface - Implemented by all 'passes'.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
The main scalar evolution driver.
StringRef - Represent a constant reference to a string, i.e.
Provides information about what library functions are available for the current target.
VPlan-based builder utility analogous to IRBuilder.
BasicBlock * getIRBasicBlock() const
Helper class to create VPRecipies from IR instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI_FOR_TEST cl::opt< bool > VerifyEachVPlan
LLVM_ABI_FOR_TEST cl::opt< bool > VPlanPrintAfterAll
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI_FOR_TEST cl::opt< bool > EnableWideActiveLaneMask
UncountableExitStyle
Different methods of handling early exits.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
LLVM_ABI_FOR_TEST cl::list< std::string > VPlanPrintAfterPasses
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
LLVM_ABI_FOR_TEST cl::opt< bool > VPlanPrintVectorRegionScope
A range of powers-of-2 vectorization factors with fixed start and adjustable end.