44 #define DEBUG_TYPE "loop-unroll"
47 "Number of loops unrolled with run-time trip counts");
67 assert(Latch &&
"Loop must have a latch");
68 BasicBlock *PrologLatch = cast<BasicBlock>(VMap[Latch]);
132 assert(Count != 0 &&
"nonsensical Count!");
141 assert(Exit &&
"Loop must have a single exit block only");
169 assert(Latch &&
"Loop must have a latch");
170 BasicBlock *EpilogLatch = cast<BasicBlock>(VMap[Latch]);
205 assert(EpilogPN->getParent() == Exit &&
"EpilogPN should be in Exit block");
217 EpilogPN->addIncoming(V, EpilogLatch);
219 assert(EpilogPN->getBasicBlockIndex(EpilogPreHeader) >= 0 &&
220 "EpilogPN should have EpilogPreHeader incoming block");
222 EpilogPN->setIncomingBlock(EpilogPN->getBasicBlockIndex(EpilogPreHeader),
246 NewExit->getFirstNonPHI());
254 PHINode *VPN = cast<PHINode>(VMap[&BBI]);
262 assert(Exit &&
"Loop must have a single exit block only");
281 const bool CreateRemainderLoop,
282 const bool UseEpilogRemainder,
285 std::vector<BasicBlock *> &NewBlocks,
288 StringRef suffix = UseEpilogRemainder ?
"epil" :
"prol";
294 Loop *NewLoop =
nullptr;
296 if (CreateRemainderLoop) {
297 NewLoop =
new Loop();
306 NewLoops[
L] = NewLoop;
308 NewLoops[
L] = ParentLoop;
314 NewBlocks.push_back(NewBB);
319 if (CreateRemainderLoop || LI->
getLoopFor(*BB) != L || ParentLoop)
332 VMap.
erase((*BB)->getTerminator());
333 BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
336 if (!CreateRemainderLoop) {
337 Builder.CreateBr(InsertBot);
346 Builder.CreateIsNotNull(IdxSub, NewIdx->
getName() +
".cmp");
347 Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
351 LatchBR->eraseFromParent();
358 PHINode *NewPHI = cast<PHINode>(VMap[&*
I]);
359 if (!CreateRemainderLoop) {
360 if (UseEpilogRemainder) {
366 cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
371 BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
388 bool IsUnrollMetadata =
false;
394 if (!IsUnrollMetadata)
451 bool AllowExpensiveTripCount,
452 bool UseEpilogRemainder,
475 if (isa<SCEVCouldNotCompute>(BECountSC) ||
482 const SCEV *TripCountSC =
484 if (isa<SCEVCouldNotCompute>(TripCountSC))
492 if (!AllowExpensiveTripCount &&
517 if (UseEpilogRemainder) {
525 DT, LI, PreserveLCSSA);
532 PrologPreHeader =
SplitEdge(PreHeader, Header, DT, LI);
569 ModVal = B.
CreateAnd(TripCount, Count - 1,
"xtraiter");
597 BasicBlock *RemainderLoop = UseEpilogRemainder ? NewExit : PrologPreHeader;
598 BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit;
600 B.
CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop);
613 std::vector<BasicBlock *> NewBlocks;
618 bool CreateRemainderLoop = (Count != 2);
623 BasicBlock *InsertBot = UseEpilogRemainder ? Exit : PrologExit;
624 BasicBlock *InsertTop = UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader;
625 CloneLoopBlocks(L, ModVal, CreateRemainderLoop, UseEpilogRemainder, InsertTop,
626 InsertBot, NewPreHeader, NewBlocks, LoopBlocks, VMap, LI);
631 NewBlocks[0]->getIterator(),
658 if (UseEpilogRemainder) {
662 EpilogPreHeader, NewPreHeader, VMap, DT, LI,
668 Value *TestVal = B2.CreateSub(TripCount, ModVal,
"unroll_iter");
670 B2.SetInsertPoint(LatchBR);
678 IdxCmp = B2.CreateIsNotNull(IdxSub, NewIdx->
getName() +
".ncmp");
680 IdxCmp = B2.CreateIsNull(IdxSub, NewIdx->
getName() +
".ncmp");
687 ConnectProlog(L, BECount, Count, PrologExit, PreHeader, NewPreHeader,
688 VMap, DT, LI, PreserveLCSSA);
696 NumRuntimeUnrolled++;
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
BasicBlock * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return an i1 value testing if Arg is not null.
const SCEV * getConstant(ConstantInt *V)
STATISTIC(NumFunctions,"Total number of functions")
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the specified block at the specified instruction - everything before SplitPt stays in Old and e...
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
static MDString * get(LLVMContext &Context, StringRef Str)
unsigned getNumOperands() const
Return number of MDNode operands.
The main scalar evolution driver.
bool isHighCostExpansion(const SCEV *Expr, Loop *L, const Instruction *At=nullptr)
Return true for expressions that may incur non-trivial cost to evaluate at runtime.
LoopT * getParentLoop() const
const Function * getParent() const
Return the enclosing method, or null if none.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
BlockT * getHeader() const
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
iterator begin()
Instruction iterator methods.
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
void setName(const Twine &Name)
Change the name of the value.
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
BasicBlock * getSuccessor(unsigned i) const
void setSuccessor(unsigned idx, BasicBlock *B)
Update the specified successor to point at the provided block.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
void perform(LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
LLVM Basic Block Representation.
This is an important class for using LLVM in a threaded context.
Type * getType() const
Return the LLVM type of this SCEV expression.
Conditional or Unconditional Branch instruction.
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
void splice(iterator where, iplist_impl &L2)
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
std::vector< BasicBlock * >::const_reverse_iterator RPOIterator
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
self_iterator getIterator()
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, BasicBlock *Exit, BasicBlock *PreHeader, BasicBlock *EpilogPreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA)
Connect the unrolling epilog code to the original loop.
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
StringRef getString() const
const MDOperand & getOperand(unsigned I) const
Iterator for intrusive lists based on ilist_node.
const BasicBlockListType & getBasicBlockList() const
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, BasicBlock *PrologExit, BasicBlock *PreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA)
Connect the unrolling prolog code to the original loop.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
pred_range predecessors(BasicBlock *BB)
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
const Loop * addClonedBlockToLoopInfo(BasicBlock *OriginalBB, BasicBlock *ClonedBB, LoopInfo *LI, NewLoopsMap &NewLoops)
Adds ClonedBB to LoopInfo, creates a new loop for ClonedBB if necessary and adds a mapping from the o...
Store the result of a depth first search within basic blocks contained by a single loop...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM...
This class uses information about analyze scalars to rewrite expressions in canonical form...
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
If this flag is set, the remapper ignores missing function-local entries (Argument, Instruction, BasicBlock) that are not in the value map.
bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, bool UseEpilogRemainder, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA)
Insert code in the prolog/epilog code when unrolling a loop with a run-time trip-count.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
This class represents an analyzed expression in the program.
Represents a single loop in the control flow graph.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool hasOneUse() const
Return true if there is exactly one user of this value.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
void setCondition(Value *V)
const SCEV * getBackedgeTakenCount(const Loop *L)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
succ_range successors(BasicBlock *BB)
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the edge connecting specified block.
StringRef - Represent a constant reference to a string, i.e.
static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop, const bool UseEpilogRemainder, BasicBlock *InsertTop, BasicBlock *InsertBot, BasicBlock *Preheader, std::vector< BasicBlock * > &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, LoopInfo *LI)
Create a clone of the blocks in a loop and connect them together.
void setIncomingValue(unsigned i, Value *V)
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
CloneBasicBlock - Return a copy of the specified basic block, but without embedding the block into a ...
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
RPOIterator endRPO() const
bool erase(const KeyT &Val)