43 #define DEBUG_TYPE "loop-unroll"
47 cl::desc(
"The baseline cost threshold for loop unrolling"));
51 cl::desc(
"The maximum 'boost' (represented as a percentage >= 100) applied "
52 "to the threshold when aggressively unrolling a loop due to the "
53 "dynamic cost savings. If completely unrolling a loop will reduce "
54 "the total runtime from X to Y, we boost the loop unroll "
55 "threshold to DefaultThreshold*std::min(MaxPercentThresholdBoost, "
56 "X/Y). This limit avoids excessive code bloat."));
60 cl::desc(
"Don't allow loop unrolling to simulate more than this number of"
61 "iterations when checking full unroll profitability"));
65 cl::desc(
"Use this unroll count for all loops including those with "
66 "unroll_count pragma values, for testing purposes"));
70 cl::desc(
"Set the max unroll count for partial and runtime unrolling, for"
76 "Set the max unroll count for full unrolling, for testing purposes"));
80 cl::desc(
"Allows loops to be partially unrolled until "
81 "-unroll-threshold loop size is reached."));
85 cl::desc(
"Allow generation of a loop remainder (extra iterations) "
86 "when unrolling a loop."));
90 cl::desc(
"Unroll loops with run-time trip counts"));
95 "The max of trip count upper bound that is considered in unrolling"));
99 cl::desc(
"Unrolled size limit for loops with an unroll(full) or "
100 "unroll_count pragma."));
104 cl::desc(
"If the runtime tripcount for the loop is lower than the "
105 "threshold, the loop is considered as flat and will be less "
106 "aggressively unrolled."));
110 cl::desc(
"Allows loops to be peeled when the dynamic "
111 "trip count is known to be low."));
150 if (L->
getHeader()->getParent()->optForSize()) {
183 UP.
Count = *UserCount;
185 UP.
Partial = *UserAllowPartial;
201 struct UnrolledInstState {
205 unsigned IsCounted : 1;
209 struct UnrolledInstStateKeyInfo {
212 static inline UnrolledInstState getEmptyKey() {
213 return {PtrInfo::getEmptyKey(), 0, 0, 0};
215 static inline UnrolledInstState getTombstoneKey() {
216 return {PtrInfo::getTombstoneKey(), 0, 0, 0};
218 static inline unsigned getHashValue(
const UnrolledInstState &S) {
219 return PairInfo::getHashValue({S.I, S.Iteration});
221 static inline bool isEqual(
const UnrolledInstState &LHS,
222 const UnrolledInstState &RHS) {
229 struct EstimatedUnrollCost {
231 unsigned UnrolledCost;
235 unsigned RolledDynamicCost;
255 unsigned MaxUnrolledLoopSize) {
260 "The unroll iterations max is too large!");
279 unsigned UnrolledCost = 0;
286 unsigned RolledDynamicCost = 0;
302 auto AddCostRecursively = [&](
Instruction &RootI,
int Iteration) {
303 assert(Iteration >= 0 &&
"Cannot have a negative iteration!");
304 assert(CostWorklist.
empty() &&
"Must start with an empty cost list");
305 assert(PHIUsedList.
empty() &&
"Must start with an empty phi used list");
307 for (;; --Iteration) {
313 auto CostIter = InstCostMap.
find({
I, Iteration, 0, 0});
314 if (CostIter == InstCostMap.
end())
319 auto &Cost = *CostIter;
325 Cost.IsCounted =
true;
328 if (
auto *PhiI = dyn_cast<PHINode>(I))
329 if (PhiI->getParent() == L->
getHeader()) {
330 assert(Cost.IsFree &&
"Loop PHIs shouldn't be evaluated as they "
331 "inherently simplify during unrolling.");
338 if (
auto *OpI = dyn_cast<Instruction>(
348 DEBUG(
dbgs() <<
"Adding cost of instruction (iteration " << Iteration
366 }
while (!CostWorklist.
empty());
368 if (PHIUsedList.
empty())
373 "Cannot track PHI-used values past the first iteration!");
383 "Must have loops in LCSSA form to track live-out values.");
385 DEBUG(
dbgs() <<
"Starting LoopUnroll profitability analysis...\n");
391 for (
unsigned Iteration = 0; Iteration < TripCount; ++Iteration) {
392 DEBUG(
dbgs() <<
" Analyzing iteration " << Iteration <<
"\n");
404 PHI->getNumIncomingValues() == 2 &&
405 "Must have an incoming value only for the preheader and the latch.");
407 Value *V = PHI->getIncomingValueForBlock(
410 if (Iteration != 0 && !C)
411 C = SimplifiedValues.
lookup(V);
413 SimplifiedInputValues.
push_back({PHI, C});
417 SimplifiedValues.
clear();
418 while (!SimplifiedInputValues.
empty())
426 for (
unsigned Idx = 0; Idx != BBWorklist.
size(); ++Idx) {
433 if (isa<DbgInfoIntrinsic>(
I))
443 bool IsFree = Analyzer.
visit(
I);
444 bool Inserted = InstCostMap.
insert({&
I, (int)Iteration,
448 assert(Inserted &&
"Cannot have a state for an unvisited instruction!");
455 if(isa<CallInst>(&
I))
460 if (
I.mayHaveSideEffects())
461 AddCostRecursively(
I, Iteration);
464 if (UnrolledCost > MaxUnrolledLoopSize) {
465 DEBUG(
dbgs() <<
" Exceeded threshold.. exiting.\n"
466 <<
" UnrolledCost: " << UnrolledCost
467 <<
", MaxUnrolledLoopSize: " << MaxUnrolledLoopSize
478 if (
BranchInst *BI = dyn_cast<BranchInst>(TI)) {
479 if (BI->isConditional()) {
481 SimplifiedValues.
lookup(BI->getCondition())) {
483 if (isa<UndefValue>(SimpleCond))
484 KnownSucc = BI->getSuccessor(0);
486 dyn_cast<ConstantInt>(SimpleCond))
487 KnownSucc = BI->getSuccessor(SimpleCondVal->isZero() ? 1 : 0);
492 SimplifiedValues.
lookup(
SI->getCondition())) {
494 if (isa<UndefValue>(SimpleCond))
495 KnownSucc =
SI->getSuccessor(0);
497 dyn_cast<ConstantInt>(SimpleCond))
498 KnownSucc =
SI->findCaseValue(SimpleCondVal).getCaseSuccessor();
503 BBWorklist.
insert(KnownSucc);
505 ExitWorklist.
insert({BB, KnownSucc});
514 ExitWorklist.
insert({BB, Succ});
515 AddCostRecursively(*TI, Iteration);
520 if (UnrolledCost == RolledDynamicCost) {
521 DEBUG(
dbgs() <<
" No opportunities found.. exiting.\n"
522 <<
" UnrolledCost: " << UnrolledCost <<
"\n");
527 while (!ExitWorklist.
empty()) {
529 std::tie(ExitingBB, ExitBB) = ExitWorklist.
pop_back_val();
536 Value *
Op = PN->getIncomingValueForBlock(ExitingBB);
537 if (
auto *OpI = dyn_cast<Instruction>(Op))
539 AddCostRecursively(*OpI, TripCount - 1);
544 <<
"UnrolledCost: " << UnrolledCost <<
", "
545 <<
"RolledDynamicCost: " << RolledDynamicCost <<
"\n");
546 return {{UnrolledCost, RolledDynamicCost}};
564 unsigned LoopSize = Metrics.
NumInsts;
572 LoopSize = std::max(LoopSize, BEInsns + 1);
613 "Unroll count hint metadata should have two operands.");
615 mdconst::extract<ConstantInt>(MD->
getOperand(1))->getZExtValue();
616 assert(Count >= 1 &&
"Unroll count must be positive.");
635 bool IsUnrollMetadata =
false;
641 if (!IsUnrollMetadata)
665 unsigned MaxPercentThresholdBoost) {
666 if (Cost.RolledDynamicCost >= UINT_MAX / 100)
668 else if (Cost.UnrolledCost != 0)
670 return std::min(100 * Cost.RolledDynamicCost / Cost.UnrolledCost,
671 MaxPercentThresholdBoost);
673 return MaxPercentThresholdBoost;
680 assert(LoopSize >= UP.
BEInsns &&
"LoopSize should not be less than BEInsns!");
689 unsigned MaxTripCount,
unsigned &TripMultiple,
unsigned LoopSize,
693 bool UserUnrollCount =
UnrollCount.getNumOccurrences() > 0;
694 if (UserUnrollCount) {
704 if (PragmaCount > 0) {
705 UP.
Count = PragmaCount;
714 if (PragmaFullUnroll && TripCount != 0) {
715 UP.
Count = TripCount;
721 bool ExplicitUnroll = PragmaCount > 0 || PragmaFullUnroll ||
722 PragmaEnableUnroll || UserUnrollCount;
724 if (ExplicitUnroll && TripCount != 0) {
741 unsigned ExactTripCount = TripCount;
742 assert((ExactTripCount == 0 || MaxTripCount == 0) &&
743 "ExtractTripCound and MaxTripCount cannot both be non zero.");
744 unsigned FullUnrollTripCount = ExactTripCount ? ExactTripCount : MaxTripCount;
745 UP.
Count = FullUnrollTripCount;
750 UseUpperBound = (MaxTripCount == FullUnrollTripCount);
751 TripCount = FullUnrollTripCount;
752 TripMultiple = UP.
UpperBound ? 1 : TripMultiple;
753 return ExplicitUnroll;
759 L, FullUnrollTripCount, DT, *SE, TTI,
763 if (Cost->UnrolledCost < UP.
Threshold * Boost / 100) {
764 UseUpperBound = (MaxTripCount == FullUnrollTripCount);
765 TripCount = FullUnrollTripCount;
766 TripMultiple = UP.
UpperBound ? 1 : TripMultiple;
767 return ExplicitUnroll;
778 DEBUG(
dbgs() <<
" will not try to unroll partially because "
779 <<
"-unroll-allow-partial not given\n");
784 UP.
Count = TripCount;
793 while (UP.
Count != 0 && TripCount % UP.
Count != 0)
801 while (UP.
Count != 0 &&
806 if (PragmaEnableUnroll)
810 <<
"Unable to unroll loop as directed by unroll(enable) pragma "
811 "because unrolled size is too large.");
815 UP.
Count = TripCount;
817 if ((PragmaFullUnroll || PragmaEnableUnroll) && TripCount &&
818 UP.
Count != TripCount)
822 <<
"Unable to fully unroll loop as directed by unroll pragma because "
823 "unrolled size is too large.");
824 return ExplicitUnroll;
827 "All cases when TripCount is constant should be covered here.");
828 if (PragmaFullUnroll)
831 "CantFullUnrollAsDirectedRuntimeTripCount",
833 <<
"Unable to fully unroll loop as directed by unroll(full) pragma "
834 "because loop has a runtime trip count.");
841 return ExplicitUnroll;
852 if (L->
getHeader()->getParent()->getEntryCount()) {
862 UP.
Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount;
864 DEBUG(
dbgs() <<
" will not try to unroll loop with runtime trip count "
865 <<
"-unroll-runtime not given\n");
874 while (UP.
Count != 0 &&
879 unsigned OrigCount = UP.
Count;
883 while (UP.
Count != 0 && TripMultiple % UP.
Count != 0)
885 DEBUG(
dbgs() <<
"Remainder loop is restricted (that could architecture "
886 "specific or because the loop contains a convergent "
887 "instruction), so unroll count must divide the trip "
889 << TripMultiple <<
". Reducing unroll count from "
890 << OrigCount <<
" to " << UP.
Count <<
".\n");
895 "DifferentUnrollCountFromDirected",
897 <<
"Unable to unroll loop the number of times directed by "
898 "unroll_count pragma because remainder loop is restricted "
899 "(that could architecture specific or because the loop "
900 "contains a convergent instruction) and so must have an unroll "
901 "count that divides the loop trip multiple of "
902 <<
NV(
"TripMultiple", TripMultiple) <<
". Unrolling instead "
903 <<
NV(
"UnrollCount", UP.
Count) <<
" time(s).");
908 DEBUG(
dbgs() <<
" partially unrolling with count: " << UP.
Count <<
"\n");
911 return ExplicitUnroll;
924 <<
"] Loop %" << L->
getHeader()->getName() <<
"\n");
929 dbgs() <<
" Not unrolling loop which is not in loop-simplify form.\n");
933 unsigned NumInlineCandidates;
934 bool NotDuplicatable;
937 L, TTI, ProvidedThreshold, ProvidedCount, ProvidedAllowPartial,
938 ProvidedRuntime, ProvidedUpperBound);
943 L, NumInlineCandidates, NotDuplicatable, Convergent, TTI, &AC, UP.
BEInsns);
944 DEBUG(
dbgs() <<
" Loop Size = " << LoopSize <<
"\n");
945 if (NotDuplicatable) {
946 DEBUG(
dbgs() <<
" Not unrolling loop which contains non-duplicatable"
947 <<
" instructions.\n");
950 if (NumInlineCandidates != 0) {
951 DEBUG(
dbgs() <<
" Not unrolling loop with inlinable calls.\n");
956 unsigned TripCount = 0;
957 unsigned MaxTripCount = 0;
958 unsigned TripMultiple = 1;
987 bool MaxOrZero =
false;
1006 bool UseUpperBound =
false;
1007 bool IsCountSetExplicitly =
1009 TripMultiple, LoopSize, UP, UseUpperBound);
1013 if (TripCount && UP.
Count > TripCount)
1014 UP.
Count = TripCount;
1019 TripMultiple, UP.
PeelCount, LI, SE, &DT, &AC, &ORE,
1027 if (IsCountSetExplicitly || UP.
PeelCount)
1034 class LoopUnroll :
public LoopPass {
1041 :
LoopPass(
ID), ProvidedCount(std::move(Count)),
1042 ProvidedThreshold(
Threshold), ProvidedAllowPartial(AllowPartial),
1043 ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound) {
1059 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1060 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1061 ScalarEvolution *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1063 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1064 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1069 bool PreserveLCSSA = mustPreserveAnalysisID(
LCSSAID);
1072 ProvidedCount, ProvidedThreshold,
1073 ProvidedAllowPartial, ProvidedRuntime,
1074 ProvidedUpperBound);
1098 int Runtime,
int UpperBound) {
1104 AllowPartial == -1 ?
None
1119 Function *
F = L.getHeader()->getParent();
1125 "cached at a higher level");
static bool tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE, const TargetTransformInfo &TTI, AssumptionCache &AC, OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, Optional< unsigned > ProvidedCount, Optional< unsigned > ProvidedThreshold, Optional< bool > ProvidedAllowPartial, Optional< bool > ProvidedRuntime, Optional< bool > ProvidedUpperBound)
Pass interface - Implemented by all 'passes'.
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U)
void push_back(const T &Elt)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PreservedAnalyses getLoopPassPreservedAnalyses()
Returns the minimum set of Analyses that all loop passes must preserve.
DiagnosticInfoOptimizationBase::Argument NV
LLVM_NODISCARD T pop_back_val()
static cl::opt< unsigned > UnrollCount("unroll-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with ""unroll_count pragma values, for testing purposes"))
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
This header provides classes for managing a pipeline of passes over loops in LLVM IR...
This is the interface for a simple mod/ref and alias analysis over globals.
bool convergent
True if this function contains a call to a convergent function.
Optional< bool > ProvidedAllowPartial
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
static MDString * get(LLVMContext &Context, StringRef Str)
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
static bool HasUnrollEnablePragma(const Loop *L)
iterator find(const ValueT &V)
Implements a dense probed hash-table based set.
unsigned getNumOperands() const
Return number of MDNode operands.
static bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE, OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount, unsigned &TripMultiple, unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound)
The main scalar evolution driver.
static unsigned getFullUnrollBoostingFactor(const EstimatedUnrollCost &Cost, unsigned MaxPercentThresholdBoost)
An immutable pass that tracks lazily created AssumptionCache objects.
static cl::opt< bool > UnrollAllowPartial("unroll-allow-partial", cl::Hidden, cl::desc("Allows loops to be partially unrolled until ""-unroll-threshold loop size is reached."))
A cache of .assume calls within a function.
static cl::opt< bool > UnrollAllowPeeling("unroll-allow-peeling", cl::Hidden, cl::desc("Allows loops to be peeled when the dynamic ""trip count is known to be low."))
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop...
unsigned NumInlineCandidates
The number of calls to internal functions with a single caller.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
bool notDuplicatable
True if this function cannot be duplicated.
size_type size() const
Determine the number of elements in the SetVector.
BlockT * getHeader() const
bool UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool Force, bool AllowRuntime, bool AllowExpensiveTripCount, bool PreserveCondBr, bool PreserveOnlyFirst, unsigned TripMultiple, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, bool PreserveLCSSA)
Unroll the given loop by Count.
static cl::opt< bool > UnrollAllowRemainder("unroll-allow-remainder", cl::Hidden, cl::desc("Allow generation of a loop remainder (extra iterations) ""when unrolling a loop."))
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static cl::opt< unsigned > UnrollMaxPercentThresholdBoost("unroll-max-percent-threshold-boost", cl::init(400), cl::Hidden, cl::desc("The maximum 'boost' (represented as a percentage >= 100) applied ""to the threshold when aggressively unrolling a loop due to the ""dynamic cost savings. If completely unrolling a loop will reduce ""the total runtime from X to Y, we boost the loop unroll ""threshold to DefaultThreshold*std::min(MaxPercentThresholdBoost, ""X/Y). This limit avoids excessive code bloat."))
static bool HasUnrollFullPragma(const Loop *L)
static cl::opt< bool > UnrollRuntime("unroll-runtime", cl::ZeroOrMore, cl::Hidden, cl::desc("Unroll loops with run-time trip counts"))
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
LLVM_NODISCARD bool empty() const
static MDNode * GetUnrollMetadataForLoop(const Loop *L, StringRef Name)
void visit(Iterator Start, Iterator End)
bool insert(const value_type &X)
Insert a new element into the SetVector.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
unsigned getSmallConstantMaxTripCount(Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
static bool isEqual(const Function &Caller, const Function &Callee)
Function Alias Analysis false
bool empty() const
Determine if the SetVector is empty or not.
void initializeLoopUnrollPass(PassRegistry &)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
iterator_range< block_iterator > blocks() const
initializer< Ty > init(const Ty &Val)
static unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls, bool &NotDuplicatable, bool &Convergent, const TargetTransformInfo &TTI, AssumptionCache *AC, unsigned BEInsns)
ApproximateLoopSize - Approximate the size of the loop.
static cl::opt< unsigned > UnrollMaxCount("unroll-max-count", cl::Hidden, cl::desc("Set the max unroll count for partial and runtime unrolling, for""testing purposes"))
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
static cl::opt< unsigned > UnrollMaxUpperBound("unroll-max-upperbound", cl::init(8), cl::Hidden, cl::desc("The max of trip count upper bound that is considered in unrolling"))
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
This is an important class for using LLVM in a threaded context.
Conditional or Unconditional Branch instruction.
static Optional< EstimatedUnrollCost > analyzeLoopUnrollCost(const Loop *L, unsigned TripCount, DominatorTree &DT, ScalarEvolution &SE, const TargetTransformInfo &TTI, unsigned MaxUnrolledLoopSize)
Figure out if the loop is worth full unrolling.
Optional< bool > ProvidedUpperBound
This is an important base class in LLVM.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
std::pair< iterator, bool > insert(const ValueT &V)
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
unsigned getSmallConstantTripCount(Loop *L)
Returns the maximum trip count of the loop if it is a single-exit loop and we can compute a small max...
Optional< unsigned > getLoopEstimatedTripCount(Loop *L)
Get a loop's estimated trip count based on branch weight metadata.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
TargetTransformInfo & TTI
void dump() const
Support for debugging, callable in GDB: V->dump()
Optional< bool > ProvidedRuntime
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
StringRef getString() const
static cl::opt< unsigned > FlatLoopTripCountThreshold("flat-loop-tripcount-threshold", cl::init(5), cl::Hidden, cl::desc("If the runtime tripcount for the loop is lower than the ""threshold, the loop is considered as flat and will be less ""aggressively unrolled."))
const MDOperand & getOperand(unsigned I) const
A SetVector that performs no allocations if smaller than a certain size.
machine trace Machine Trace Metrics
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
bool isLCSSAForm(DominatorTree &DT) const
Return true if the Loop is in LCSSA form.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(Loop *L, const TargetTransformInfo &TTI, Optional< unsigned > UserThreshold, Optional< unsigned > UserCount, Optional< bool > UserAllowPartial, Optional< bool > UserRuntime, Optional< bool > UserUpperBound)
Gather the various unrolling parameters based on the defaults, compiler flags, TTI overrides and user...
unsigned getSmallConstantTripMultiple(Loop *L)
Returns the largest constant divisor of the trip count of the loop if it is a single-exit loop and we...
Utility to calculate the size and a few similar metrics for a set of basic blocks.
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static unsigned UnrollCountPragmaValue(const Loop *L)
LLVM_NODISCARD T pop_back_val()
Optional< unsigned > ProvidedThreshold
Pass * createLoopUnrollPass(int Threshold=-1, int Count=-1, int AllowPartial=-1, int Runtime=-1, int UpperBound=-1)
static cl::opt< unsigned > PragmaUnrollThreshold("pragma-unroll-threshold", cl::init(16 *1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll(full) or ""unroll_count pragma."))
static uint64_t getUnrolledLoopSize(unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
static cl::opt< unsigned > UnrollFullMaxCount("unroll-full-max-count", cl::Hidden, cl::desc("Set the max unroll count for full unrolling, for testing purposes"))
static void SetLoopAlreadyUnrolled(Loop *L)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void clear()
Completely clear the SetVector.
static bool HasUnrollDisablePragma(const Loop *L)
MDNode * GetUnrollMetadata(MDNode *LoopID, StringRef Name)
Given an llvm.loop loop id metadata node, returns the loop hint metadata node with the given name (fo...
static cl::opt< unsigned > UnrollMaxIterationsCountToAnalyze("unroll-max-iteration-count-to-analyze", cl::init(10), cl::Hidden, cl::desc("Don't allow loop unrolling to simulate more than this number of""iterations when checking full unroll profitability"))
Optional< unsigned > ProvidedCount
static const unsigned NoThreshold
A magic value for use with the Threshold parameter to indicate that the loop unroll should be perform...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Represents a single loop in the control flow graph.
static cl::opt< unsigned > UnrollThreshold("unroll-threshold", cl::Hidden, cl::desc("The baseline cost threshold for loop unrolling"))
void getLoopAnalysisUsage(AnalysisUsage &AU)
Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
void computePeelCount(Loop *L, unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)
void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &EphValues)
Add information about a block to the current state.
static int const Threshold
TODO: Write a new FunctionPass AliasAnalysis so that it can keep a cache.
static bool HasRuntimeUnrollDisablePragma(const Loop *L)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Pass * createSimpleLoopUnrollPass()
succ_range successors(BasicBlock *BB)
bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getMaxBackedgeTakenCount or z...
StringRef - Represent a constant reference to a string, i.e.
A container for analyses that lazily runs them and caches their results.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop)...
unsigned NumInsts
Number of instructions in the analyzed blocks.