37 #define DEBUG_TYPE "loop-unroll"
38 STATISTIC(NumPeeled,
"Number of loops peeled");
42 cl::desc(
"Max average trip count which will cause loop peeling."));
46 cl::desc(
"Force a peel count regardless of profiling information."));
91 DEBUG(
dbgs() <<
"Profile-based estimated trip count is " << *PeelCount
96 (LoopSize * (*PeelCount + 1) <= UP.
Threshold)) {
97 DEBUG(
dbgs() <<
"Peeling first " << *PeelCount <<
" iterations.\n");
101 DEBUG(
dbgs() <<
"Requested peel count: " << *PeelCount <<
"\n");
103 DEBUG(
dbgs() <<
"Peel cost: " << LoopSize * (*PeelCount + 1) <<
"\n");
131 unsigned IterNumber,
unsigned AvgIters,
132 uint64_t &PeeledHeaderWeight) {
138 if (PeeledHeaderWeight) {
139 uint64_t FallThruWeight =
140 PeeledHeaderWeight * ((float)(AvgIters - IterNumber) / AvgIters * 0.9);
141 uint64_t ExitWeight = PeeledHeaderWeight - FallThruWeight;
142 PeeledHeaderWeight -= ExitWeight;
144 unsigned HeaderIdx = (LatchBR->
getSuccessor(0) == Header ? 0 : 1);
147 HeaderIdx ? MDB.createBranchWeights(ExitWeight, FallThruWeight)
148 : MDB.createBranchWeights(FallThruWeight, ExitWeight);
202 cast<BranchInst>(cast<BasicBlock>(VMap[Latch])->getTerminator());
203 unsigned HeaderIdx = (LatchBR->
getSuccessor(0) == Header ? 0 : 1);
216 PHINode *NewPHI = cast<PHINode>(VMap[&*
I]);
217 if (IterNumber == 0) {
222 if (LatchInst && L->
contains(LatchInst))
223 VMap[&*
I] = LVMap[LatchInst];
225 VMap[&*
I] = LatchVal;
227 cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
238 if (LatchInst && L->
contains(LatchInst))
239 LatchVal = VMap[LatchVal];
240 PHI->
addIncoming(LatchVal, cast<BasicBlock>(VMap[Latch]));
245 for (
const auto &KV : VMap)
246 LVMap[KV.first] = KV.second;
260 bool PreserveLCSSA) {
327 NewPreHeader->setName(PreHeader->
getName() +
".peel.newph");
334 cast<BranchInst>(cast<BasicBlock>(Latch)->getTerminator());
335 unsigned HeaderIdx = (LatchBR->
getSuccessor(0) == Header ? 0 : 1);
337 uint64_t TrueWeight, FalseWeight;
338 uint64_t ExitWeight = 0, CurHeaderWeight = 0;
340 ExitWeight = HeaderIdx ? TrueWeight : FalseWeight;
343 CurHeaderWeight = TrueWeight + FalseWeight;
347 for (
unsigned Iter = 0; Iter < PeelCount; ++Iter) {
355 if (ExitWeight < CurHeaderWeight)
356 CurHeaderWeight -= ExitWeight;
361 NewBlocks, LoopBlocks, VMap, LVMap, LI);
363 PeelCount, ExitWeight);
365 InsertTop = InsertBot;
371 NewBlocks[0]->getIterator(), F->
end());
384 if (LatchInst && L->
contains(LatchInst))
385 NewVal = LVMap[LatchInst];
395 uint64_t BackEdgeWeight = 0;
396 if (ExitWeight < CurHeaderWeight)
397 BackEdgeWeight = CurHeaderWeight - ExitWeight;
402 HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight)
403 : MDB.createBranchWeights(BackEdgeWeight, ExitWeight);
void push_back(const T &Elt)
BasicBlock * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
STATISTIC(NumFunctions,"Total number of functions")
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the specified block at the specified instruction - everything before SplitPt stays in Old and e...
static void cloneLoopBlocks(Loop *L, unsigned IterNumber, BasicBlock *InsertTop, BasicBlock *InsertBot, BasicBlock *Exit, SmallVectorImpl< BasicBlock * > &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, LoopInfo *LI)
Clones the body of the loop L, putting it between InsertTop and InsertBot.
The main scalar evolution driver.
LoopT * getParentLoop() const
const Function * getParent() const
Return the enclosing method, or null if none.
BlockT * getHeader() const
void remapInstructionsInBlocks(const SmallVectorImpl< BasicBlock * > &Blocks, ValueToValueMapTy &VMap)
Remaps instructions in Blocks using the mapping in VMap.
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
iterator begin()
Instruction iterator methods.
static cl::opt< unsigned > UnrollPeelMaxCount("unroll-peel-max-count", cl::init(7), cl::Hidden, cl::desc("Max average trip count which will cause loop peeling."))
static void updateBranchWeights(BasicBlock *Header, BranchInst *LatchBR, unsigned IterNumber, unsigned AvgIters, uint64_t &PeeledHeaderWeight)
Update the branch weights of the latch of a peeled-off loop iteration.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void setName(const Twine &Name)
Change the name of the value.
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
BasicBlock * getSuccessor(unsigned i) const
void setSuccessor(unsigned idx, BasicBlock *B)
Update the specified successor to point at the provided block.
void perform(LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
initializer< Ty > init(const Ty &Val)
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
Conditional or Unconditional Branch instruction.
static cl::opt< unsigned > UnrollForcePeelCount("unroll-force-peel-count", cl::init(0), cl::Hidden, cl::desc("Force a peel count regardless of profiling information."))
bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) const
Retrieve the raw weight values of a conditional branch or select.
void splice(iterator where, iplist_impl &L2)
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
std::vector< BasicBlock * >::const_reverse_iterator RPOIterator
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
self_iterator getIterator()
Optional< unsigned > getLoopEstimatedTripCount(Loop *L)
Get a loop's estimated trip count based on branch weight metadata.
LLVMContext & getContext() const
All values hold a context through their type.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Iterator for intrusive lists based on ilist_node.
const BasicBlockListType & getBasicBlockList() const
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
static bool canPeel(Loop *L)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Store the result of a depth first search within basic blocks contained by a single loop...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA)
Peel off the first PeelCount iterations of loop L.
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
Represents a single loop in the control flow graph.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
void computePeelCount(Loop *L, unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
LLVM Value Representation.
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the edge connecting specified block.
void setIncomingValue(unsigned i, Value *V)
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
CloneBasicBlock - Return a copy of the specified basic block, but without embedding the block into a ...
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
RPOIterator endRPO() const