41 #define DEBUG_TYPE "loop-unroll"
44 STATISTIC(NumCompletelyUnrolled,
"Number of loops completely unrolled");
45 STATISTIC(NumUnrolled,
"Number of loops unrolled (completely or otherwise)");
58 if (
PHINode *PN = dyn_cast<PHINode>(I)) {
59 for (
unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
62 PN->setIncomingBlock(i, cast<BasicBlock>(It->
second));
82 if (!OnlyPred)
return nullptr;
87 DEBUG(
dbgs() <<
"Merging: " << *BB <<
"into: " << *OnlyPred);
115 if (ForgottenLoops.
insert(L).second)
164 bool AllowRuntime,
bool AllowExpensiveTripCount,
169 DEBUG(
dbgs() <<
" Can't unroll; loop preheader-insertion failed.\n");
175 DEBUG(
dbgs() <<
" Can't unroll; loop exit-block-insertion failed.\n");
181 DEBUG(
dbgs() <<
" Can't unroll; Loop body cannot be cloned.\n");
191 " Can't unroll; loop not terminated by a conditional branch.\n");
198 " Won't unroll loop: address of header block is taken.\n");
203 DEBUG(
dbgs() <<
" Trip Count = " << TripCount <<
"\n");
204 if (TripMultiple != 1)
205 DEBUG(
dbgs() <<
" Trip Multiple = " << TripMultiple <<
"\n");
209 if (TripCount != 0 && Count > TripCount)
214 if (TripCount == 0 && Count < 2)
218 assert(TripMultiple > 0);
219 assert(TripCount == 0 || TripCount % TripMultiple == 0);
222 bool CompletelyUnroll = Count == TripCount;
227 bool RuntimeTripCount = (TripCount == 0 && Count > 0 && AllowRuntime);
229 if (RuntimeTripCount &&
241 unsigned BreakoutTrip = 0;
242 if (TripCount != 0) {
243 BreakoutTrip = TripCount % Count;
247 BreakoutTrip = TripMultiple =
256 if (CompletelyUnroll) {
258 <<
" with trip count " << TripCount <<
"!\n");
260 Twine(
"completely unrolled loop with ") +
261 Twine(TripCount) +
" iterations");
263 auto EmitDiag = [&](
const Twine &
T) {
265 "unrolled loop by a factor of " +
Twine(Count) +
271 if (TripMultiple == 0 || BreakoutTrip != TripMultiple) {
272 DEBUG(
dbgs() <<
" with a breakout at trip " << BreakoutTrip);
273 EmitDiag(
" with a breakout at trip " +
Twine(BreakoutTrip));
274 }
else if (TripMultiple != 1) {
275 DEBUG(
dbgs() <<
" with " << TripMultiple <<
" trips per branch");
276 EmitDiag(
" with " +
Twine(TripMultiple) +
" trips per branch");
277 }
else if (RuntimeTripCount) {
278 DEBUG(
dbgs() <<
" with run-time trip count");
279 EmitDiag(
" with run-time trip count");
290 std::vector<PHINode*> OrigPHINode;
292 OrigPHINode.push_back(cast<PHINode>(
I));
295 std::vector<BasicBlock*> Headers;
296 std::vector<BasicBlock*> Latches;
297 Headers.push_back(Header);
298 Latches.push_back(LatchBlock);
310 for (
unsigned It = 1; It != Count; ++It) {
311 std::vector<BasicBlock*> NewBlocks;
322 assert(LI->
getLoopFor(*BB) == L &&
"Header should not be in a sub-loop");
327 assert(OldLoop &&
"Should (at least) be in the loop being unrolled!");
329 Loop *&NewLoop = NewLoops[OldLoop];
333 "Header should be first in RPO");
336 assert(NewLoopParent &&
337 "Expected parent loop before sub-loop in RPO");
351 for (
unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
352 PHINode *NewPHI = cast<PHINode>(VMap[OrigPHINode[i]]);
354 if (
Instruction *InValI = dyn_cast<Instruction>(InVal))
356 InVal = LastValueMap[InValI];
357 VMap[OrigPHINode[i]] = InVal;
362 LastValueMap[*BB] = New;
365 LastValueMap[
VI->first] =
VI->second;
376 if (It != LastValueMap.
end())
384 Headers.push_back(New);
385 if (*BB == LatchBlock)
386 Latches.push_back(New);
388 NewBlocks.push_back(New);
392 for (
unsigned i = 0; i < NewBlocks.size(); ++i)
394 E = NewBlocks[i]->
end();
I != E; ++
I)
399 for (
unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
401 if (CompletelyUnroll) {
405 else if (Count > 1) {
409 if (
Instruction *InValI = dyn_cast<Instruction>(InVal)) {
411 InVal = LastValueMap[InVal];
413 assert(Latches.back() == LastValueMap[LatchBlock] &&
"bad last latch");
420 for (
unsigned i = 0, e = Latches.size(); i != e; ++i) {
422 BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
425 unsigned j = (i + 1) % e;
427 bool NeedConditional =
true;
429 if (RuntimeTripCount && j != 0) {
430 NeedConditional =
false;
435 if (CompletelyUnroll && j == 0) {
437 NeedConditional =
false;
442 if (j != BreakoutTrip && (TripMultiple == 0 || j % TripMultiple != 0)) {
443 NeedConditional =
false;
446 if (NeedConditional) {
452 if (Dest != LoopExit) {
456 if (*
SI == Headers[i])
472 for (
unsigned i = 0, e = Latches.size(); i != e; ++i) {
473 BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
478 std::replace(Latches.begin(), Latches.end(), Dest, Fold);
492 DT = &DTWP->getDomTree();
497 if (SE && !CompletelyUnroll) {
503 while (!DeadInsts.
empty())
505 dyn_cast_or_null<Instruction>(&*DeadInsts.
pop_back_val()))
513 const std::vector<BasicBlock*> &NewLoopBlocks = L->
getBlocks();
514 for (std::vector<BasicBlock*>::const_iterator BB = NewLoopBlocks.begin(),
515 BBE = NewLoopBlocks.end(); BB != BBE; ++BB)
520 (*BB)->getInstList().erase(Inst);
524 (*BB)->getInstList().erase(Inst);
528 NumCompletelyUnrolled += CompletelyUnroll;
533 if (CompletelyUnroll && LPM !=
nullptr)
541 if (!OuterL && !CompletelyUnroll)
566 assert(LoopID->
getNumOperands() > 0 &&
"requires at least one operand");
567 assert(LoopID->
getOperand(0) == LoopID &&
"invalid loop id");
Pass interface - Implemented by all 'passes'.
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
A parsed version of the target data layout string in and methods for querying it. ...
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
void removeBlock(BlockT *BB)
removeBlock - This method completely removes BB from all data structures, including all of the Loop o...
uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B)
GreatestCommonDivisor64 - Return the greatest common divisor of the two values using Euclid's algorit...
STATISTIC(NumFunctions,"Total number of functions")
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
unsigned getNumOperands() const
unsigned getNumOperands() const
Return number of MDNode operands.
ScalarEvolution - This class is the main scalar evolution driver.
A cache of .assume calls within a function.
const_iterator begin(StringRef path)
Get begin iterator over path.
LoopT * getParentLoop() const
const Function * getParent() const
Return the enclosing method, or null if none.
LoopT * getLoopFor(const BlockT *BB) const
getLoopFor - Return the inner most loop that BB lives in.
static BasicBlock * FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo *LI, LPPassManager *LPM, SmallPtrSetImpl< Loop * > &ForgottenLoops)
FoldBlockIntoPredecessor - Folds a basic block into its predecessor if it only has one predecessor...
const std::vector< BlockT * > & getBlocks() const
getBlocks - Get a list of the basic blocks which make up this loop.
void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg)
Emit an optimization-applied message.
BlockT * getHeader() const
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
getLoopLatch - If there is a single latch block for this loop, return it.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
bool isUnconditional() const
void push_back(NodeTy *val)
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
removeIncomingValue - Remove an incoming value.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
AnalysisType * getAnalysisIfAvailable() const
getAnalysisIfAvailable<AnalysisType>() - Subclasses use this function to get analysis information tha...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP, AliasAnalysis *AA=nullptr, ScalarEvolution *SE=nullptr, AssumptionCache *AC=nullptr)
Simplify each loop in a loop nest recursively.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
void setName(const Twine &Name)
Change the name of the value.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
addBasicBlockToLoop - This method is used by other analyses to update loop information.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
void addChildLoop(LoopT *NewChild)
addChildLoop - Add the specified loop to be a child of this loop.
BasicBlock * getSuccessor(unsigned i) const
iterator find(const KeyT &Val)
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
void perform(LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Interval::succ_iterator succ_end(Interval *I)
void FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA=nullptr, MemoryDependenceAnalysis *MemDep=nullptr)
FoldSingleEntryPHINodes - We know that BB has one predecessor.
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
void clear()
Clear the cache of .assume intrinsics for a function.
friend const_iterator end(StringRef path)
Get end iterator over path.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
This is an important class for using LLVM in a threaded context.
BranchInst - Conditional or Unconditional Branch instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, LPPassManager *LPM, SmallVectorImpl< WeakVH > &Dead)
SimplifyLoopIVs - Simplify users of induction variables within this loop.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
const InstListType & getInstList() const
Return the underlying instruction list container.
bool isSafeToClone() const
isSafeToClone - Return true if the loop body is safe to clone in practice.
std::vector< BasicBlock * >::const_reverse_iterator RPOIterator
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
Value * getOperand(unsigned i) const
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
iterator erase(iterator where)
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
StringRef getString() const
const MDOperand & getOperand(unsigned I) const
const BasicBlockListType & getBasicBlockList() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
void deleteLoopFromQueue(Loop *L)
Delete loop from the loop queue and loop hierarchy (LoopInfo).
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void splice(iterator where, iplist &L2)
void setOperand(unsigned i, Value *Val)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Store the result of a depth first search within basic blocks contained by a single loop...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
RemapInstruction - Convert the instruction operands from referencing the current values into those sp...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
MDNode * GetUnrollMetadata(MDNode *LoopID, StringRef Name)
Given an llvm.loop loop id metadata node, returns the loop hint metadata node with the given name (fo...
bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE=nullptr)
Put a loop nest into LCSSA form.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void forgetLoop(const Loop *L)
forgetLoop - This method should be called by the client when it has changed a loop in a way that may ...
iplist< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
isInstructionTriviallyDead - Return true if the result produced by the instruction is not used...
LLVM Value Representation.
void recalculate(FT &F)
recalculate - compute a dominator tree for the given function
StringRef - Represent a constant reference to a string, i.e.
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
SimplifyInstruction - See if we can compute a simplified version of this instruction.
Legacy analysis pass which computes a DominatorTree.
bool UnrollRuntimeLoopProlog(Loop *L, unsigned Count, bool AllowExpensiveTripCount, LoopInfo *LI, LPPassManager *LPM)
Insert code in the prolog code when unrolling a loop with a run-time trip-count.
bool replacementPreservesLCSSAForm(Instruction *From, Value *To)
replacementPreservesLCSSAForm - Returns true if replacing From with To everywhere is guaranteed to pr...
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
CloneBasicBlock - Return a copy of the specified basic block, but without embedding the block into a ...
bool UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool AllowRuntime, bool AllowExpensiveTripCount, unsigned TripMultiple, LoopInfo *LI, Pass *PP, LPPassManager *LPM, AssumptionCache *AC)
Unroll the given loop by Count.
RPOIterator endRPO() const
bool empty() const
empty - Check if the string is empty.