37 #define DEBUG_TYPE "loop-rotate"
41 cl::desc(
"The default maximum header size for automatic loop rotation"));
43 STATISTIC(NumRotated,
"Number of loops rotated");
49 LoopRotate(
int SpecifiedMaxHeaderSize = -1) :
LoopPass(
ID) {
51 if (SpecifiedMaxHeaderSize == -1)
54 MaxHeaderSize =
unsigned(SpecifiedMaxHeaderSize);
72 bool simplifyLoopLatch(
Loop *L);
73 bool rotateLoop(
Loop *L,
bool SimplifiedLatch);
76 unsigned MaxHeaderSize;
94 return new LoopRotate(MaxHeaderSize);
100 if (skipOptnoneFunction(L))
108 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
109 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
110 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
111 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
112 DT = DTWP ? &DTWP->getDomTree() :
nullptr;
117 bool SimplifiedLatch = simplifyLoopLatch(L);
120 bool MadeChange =
false;
121 while (rotateLoop(L, SimplifiedLatch)) {
123 SimplifiedLatch =
false;
128 if ((MadeChange || SimplifiedLatch) && LoopMD)
149 for (I = OrigHeader->
begin(); I != E; ++
I) {
157 Value *OrigPreHeaderVal = ValueMap[OrigHeaderVal];
167 UE = OrigHeaderVal->
use_end(); UI != UE; ) {
177 if (!isa<PHINode>(UserInst)) {
182 if (UserBB == OrigHeader)
187 if (UserBB == OrigPreheader) {
188 U = OrigPreHeaderVal;
205 bool seenIncrement =
false;
206 bool MultiExitLoop =
false;
209 MultiExitLoop =
true;
216 if (isa<DbgInfoIntrinsic>(
I))
219 switch (
I->getOpcode()) {
222 case Instruction::GetElementPtr:
224 if (!cast<GEPOperator>(
I)->hasAllConstantIndices())
227 case Instruction::Add:
228 case Instruction::Sub:
232 case Instruction::Shl:
233 case Instruction::LShr:
234 case Instruction::AShr: {
235 Value *IVOpnd = !isa<Constant>(
I->getOperand(0))
237 : !isa<Constant>(
I->getOperand(1))
247 auto *UserInst = cast<Instruction>(UseI);
255 seenIncrement =
true;
258 case Instruction::Trunc:
259 case Instruction::ZExt:
260 case Instruction::SExt:
276 bool LoopRotate::simplifyLoopLatch(
Loop *L) {
297 << LastExit->
getName() <<
"\n");
302 unsigned FallThruPath = BI->
getSuccessor(0) == Latch ? 0 : 1;
304 assert(Header == L->
getHeader() &&
"expected a backward branch");
312 assert(Latch->
empty() &&
"unable to evacuate Latch");
313 LI->removeBlock(Latch);
315 DT->eraseNode(Latch);
330 bool LoopRotate::rotateLoop(
Loop *L,
bool SimplifiedLatch) {
367 DEBUG(
dbgs() <<
"LoopRotation: NOT rotating - contains non-duplicatable"
368 <<
" instructions: "; L->
dump());
371 if (Metrics.
NumInsts > MaxHeaderSize)
397 assert(NewHeader &&
"Unable to determine new loop header");
399 "Unable to determine loop header and exit blocks");
404 "New header doesn't have one pred!");
433 !isa<TerminatorInst>(Inst) && !isa<DbgInfoIntrinsic>(Inst) &&
434 !isa<AllocaInst>(Inst)) {
451 if (V && LI->replacementPreservesLCSSAForm(C, V)) {
484 assert(L->
getHeader() == NewHeader &&
"Latch block is our new header");
495 assert(PHBI->isConditional() &&
"Should be clone of BI condbr!");
496 if (!isa<ConstantInt>(PHBI->getCondition()) ||
497 PHBI->getSuccessor(cast<ConstantInt>(PHBI->getCondition())->
isZero())
507 DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
509 OrigHeaderNode->
end());
510 DomTreeNode *OrigPreheaderNode = DT->getNode(OrigPreheader);
511 for (
unsigned I = 0, E = HeaderChildren.size(); I != E; ++
I)
512 DT->changeImmediateDominator(HeaderChildren[I], OrigPreheaderNode);
514 assert(DT->getNode(Exit)->getIDom() == OrigPreheaderNode);
515 assert(DT->getNode(NewHeader)->getIDom() == OrigPreheaderNode);
518 DT->changeImmediateDominator(OrigHeader, OrigLatch);
525 OrigPreheader, NewHeader,
534 bool SplitLatchEdge =
false;
536 PE = ExitPreds.end();
539 Loop *PredLoop = LI->getLoopFor(*PI);
540 if (!PredLoop || PredLoop->
contains(Exit))
542 if (isa<IndirectBrInst>((*PI)->getTerminator()))
549 assert(SplitLatchEdge &&
550 "Despite splitting all preds, failed to split latch exit?");
554 Exit->removePredecessor(OrigPreheader,
true );
557 PHBI->eraseFromParent();
562 DT->changeImmediateDominator(NewHeader, OrigPreheader);
563 DT->changeImmediateDominator(OrigHeader, OrigLatch);
568 DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
570 OrigHeaderNode->
end());
574 for (
unsigned I = 0, E = HeaderChildren.size(); I != E; ++
I) {
581 NearestDom = DT->findNearestCommonDominator(NearestDom, *PI);
585 DT->changeImmediateDominator(BB, NearestDom);
596 assert(L->
getLoopPreheader() &&
"Invalid loop preheader after loop rotation");
597 assert(L->
getLoopLatch() &&
"Invalid loop latch after loop rotation");
Pass interface - Implemented by all 'passes'.
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
use_iterator_impl< Use > use_iterator
A parsed version of the target data layout string in and methods for querying it. ...
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Helper class for SSA formation on a set of values defined in multiple blocks.
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
void Initialize(Type *Ty, StringRef Name)
Reset this object to get ready for a new set of SSA updates with type 'Ty'.
void AddAvailableValue(BasicBlock *BB, Value *V)
Indicate that a rewritten value is available in the specified block with the specified value...
ScalarEvolution - This class is the main scalar evolution driver.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of .assume calls within a function.
bool isLoopExiting(const BlockT *BB) const
isLoopExiting - True if terminator in the block can branch to another block that is outside of the cu...
bool hasLoopInvariantOperands(const Instruction *I) const
hasLoopInvariantOperands - Return true if all the operands of the specified instruction are loop inva...
bool notDuplicatable
True if this function cannot be duplicated.
const std::vector< BlockT * > & getBlocks() const
getBlocks - Get a list of the basic blocks which make up this loop.
BlockT * getHeader() const
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
getLoopLatch - If there is a single latch block for this loop, return it.
iterator begin()
Instruction iterator methods.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
bool isUnconditional() const
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
removeIncomingValue - Remove an incoming value.
Option class for critical edge splitting.
A Use represents the edge between a Value definition and its users.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
void setName(const Twine &Name)
Change the name of the value.
Instruction * clone() const
clone() - Create a copy of 'this' instruction that is identical in all ways except the following: ...
bool mayReadFromMemory() const
mayReadFromMemory - Return true if this instruction may read memory.
bool MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, AliasAnalysis *AA=nullptr, MemoryDependenceAnalysis *MemDep=nullptr)
MergeBlockIntoPredecessor - Attempts to merge a block into its predecessor, if possible.
BasicBlock * getSuccessor(unsigned i) const
Base class for the actual dominator tree node.
uint64_t rotate(uint64_t val, size_t shift)
Bitwise right rotate.
void initializeLoopRotatePass(PassRegistry &)
AnalysisUsage & addPreservedID(const void *ID)
static bool shouldSpeculateInstrs(BasicBlock::iterator Begin, BasicBlock::iterator End, Loop *L)
Determine whether the instructions in this range may be safely and cheaply speculated.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
RF_NoModuleLevelChanges - If this flag is set, the remapper knows that only local values within a fun...
RF_IgnoreMissingEntries - If this flag is set, the remapper ignores entries that are not in the value...
void FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA=nullptr, MemoryDependenceAnalysis *MemDep=nullptr)
FoldSingleEntryPHINodes - We know that BB has one predecessor.
void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI, SmallPtrSetImpl< const Value * > &EphValues)
Add information about a block to the current state.
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
void replaceSuccessorsPhiUsesWith(BasicBlock *New)
Update all phi nodes in this basic block's successors to refer to basic block New instead of to it...
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
SplitCriticalEdge - If this edge is a critical edge, insert a new node to split the critical edge...
initializer< Ty > init(const Ty &Val)
Pass * createLoopRotatePass(int MaxHeaderSize=-1)
Subclasses of this class are all able to terminate a basic block.
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
LLVM Basic Block Representation.
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader, BasicBlock *OrigPreheader, ValueToValueMapTy &ValueMap)
RewriteUsesOfClonedInstructions - We just cloned the instructions from the old header into the prehea...
BranchInst - Conditional or Unconditional Branch instruction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
const InstListType & getInstList() const
Return the underlying instruction list container.
User * getUser() const
Returns the User that contains this Use.
BlockT * getExitingBlock() const
getExitingBlock - If getExitingBlocks would return exactly one block, return that block...
Interval::pred_iterator pred_end(Interval *I)
bool mayWriteToMemory() const
mayWriteToMemory - Return true if this instruction may modify memory.
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
machine trace Machine Trace Metrics
DomTreeNodeBase< NodeT > * getIDom() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
AnalysisUsage & addRequiredID(const void *ID)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Utility to calculate the size and a few similar metrics for a set of basic blocks.
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void splice(iterator where, iplist &L2)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
RemapInstruction - Convert the instruction operands from referencing the current values into those sp...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
void moveToHeader(BlockT *BB)
moveToHeader - This method is used to move BB (which must be part of this loop) to be the loop header...
iplist< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM Value Representation.
void moveBefore(Instruction *MovePos)
moveBefore - Unlink this instruction from its current basic block and insert it into the basic block ...
The legacy pass manager's analysis pass to compute loop information.
C - The default llvm calling convention, compatible with C.
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
SimplifyInstruction - See if we can compute a simplified version of this instruction.
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
isSafeToSpeculativelyExecute - Return true if the instruction does not have any effects besides calcu...
Legacy analysis pass which computes a DominatorTree.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop)...
unsigned NumInsts
Number of instructions in the analyzed blocks.
void RewriteUse(Use &U)
Rewrite a use of the symbolic value.
int getBasicBlockIndex(const BasicBlock *BB) const
getBasicBlockIndex - Return the first index of the specified basic block in the value list for this P...
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const BasicBlock * getParent() const
static cl::opt< unsigned > DefaultRotationThreshold("rotation-max-header-size", cl::init(16), cl::Hidden, cl::desc("The default maximum header size for automatic loop rotation"))