44 #define DEBUG_TYPE "loop-rotate"
48 cl::desc(
"The default maximum header size for automatic loop rotation"));
50 STATISTIC(NumRotated,
"Number of loops rotated");
55 const unsigned MaxHeaderSize;
63 LoopRotate(
unsigned MaxHeaderSize,
LoopInfo *LI,
66 : MaxHeaderSize(MaxHeaderSize), LI(LI), TTI(TTI), AC(AC), DT(DT), SE(SE) {
68 bool processLoop(
Loop *
L);
71 bool rotateLoop(
Loop *
L,
bool SimplifiedLatch);
72 bool simplifyLoopLatch(
Loop *
L);
91 for (I = OrigHeader->
begin(); I !=
E; ++
I) {
92 Value *OrigHeaderVal = &*
I;
99 Value *OrigPreHeaderVal = ValueMap.
lookup(OrigHeaderVal);
120 if (!isa<PHINode>(UserInst)) {
125 if (UserBB == OrigHeader)
130 if (UserBB == OrigPreheader) {
131 U = OrigPreHeaderVal;
145 for (
auto UI = MAV->use_begin(), E = MAV->use_end(); UI !=
E;) {
156 if (UserBB == OrigHeader)
164 if (UserBB == OrigPreheader)
165 NewVal = OrigPreHeaderVal;
187 bool LoopRotate::rotateLoop(
Loop *
L,
bool SimplifiedLatch) {
224 DEBUG(
dbgs() <<
"LoopRotation: NOT rotating - contains non-duplicatable"
225 <<
" instructions: ";
230 DEBUG(
dbgs() <<
"LoopRotation: NOT rotating - contains convergent "
235 if (Metrics.
NumInsts > MaxHeaderSize)
261 assert(NewHeader &&
"Unable to determine new loop header");
263 "Unable to determine loop header and exit blocks");
268 "New header doesn't have one pred!");
297 !isa<DbgInfoIntrinsic>(Inst) && !isa<AllocaInst>(Inst)) {
314 if (V && LI->replacementPreservesLCSSAForm(C, V)) {
330 if (
auto *II = dyn_cast<IntrinsicInst>(C))
331 if (II->getIntrinsicID() == Intrinsic::assume)
332 AC->registerAssumption(II);
356 assert(L->
getHeader() == NewHeader &&
"Latch block is our new header");
366 assert(PHBI->isConditional() &&
"Should be clone of BI condbr!");
367 if (!isa<ConstantInt>(PHBI->getCondition()) ||
368 PHBI->getSuccessor(cast<ConstantInt>(PHBI->getCondition())->
isZero()) !=
378 DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
380 OrigHeaderNode->
end());
381 DomTreeNode *OrigPreheaderNode = DT->getNode(OrigPreheader);
382 for (
unsigned I = 0,
E = HeaderChildren.size(); I !=
E; ++
I)
383 DT->changeImmediateDominator(HeaderChildren[I], OrigPreheaderNode);
385 assert(DT->getNode(Exit)->getIDom() == OrigPreheaderNode);
386 assert(DT->getNode(NewHeader)->getIDom() == OrigPreheaderNode);
389 DT->changeImmediateDominator(OrigHeader, OrigLatch);
396 OrigPreheader, NewHeader,
405 bool SplitLatchEdge =
false;
408 Loop *PredLoop = LI->getLoopFor(ExitPred);
409 if (!PredLoop || PredLoop->
contains(Exit))
411 if (isa<IndirectBrInst>(ExitPred->getTerminator()))
420 "Despite splitting all preds, failed to split latch exit?");
424 Exit->removePredecessor(OrigPreheader,
true );
427 PHBI->eraseFromParent();
432 DT->changeImmediateDominator(NewHeader, OrigPreheader);
433 DT->changeImmediateDominator(OrigHeader, OrigLatch);
438 DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
440 OrigHeaderNode->
end());
444 for (
unsigned I = 0,
E = HeaderChildren.size(); I !=
E; ++
I) {
451 NearestDom = DT->findNearestCommonDominator(NearestDom, *PI);
454 if (Node->
getIDom()->getBlock() != NearestDom) {
455 DT->changeImmediateDominator(BB, NearestDom);
487 bool seenIncrement =
false;
488 bool MultiExitLoop =
false;
491 MultiExitLoop =
true;
498 if (isa<DbgInfoIntrinsic>(I))
501 switch (I->getOpcode()) {
504 case Instruction::GetElementPtr:
506 if (!cast<GEPOperator>(I)->hasAllConstantIndices())
511 case Instruction::Sub:
515 case Instruction::Shl:
516 case Instruction::LShr:
517 case Instruction::AShr: {
519 !isa<Constant>(I->getOperand(0))
521 : !isa<Constant>(I->getOperand(1)) ? I->getOperand(1) :
nullptr;
529 auto *UserInst = cast<Instruction>(UseI);
537 seenIncrement =
true;
540 case Instruction::Trunc:
541 case Instruction::ZExt:
542 case Instruction::SExt:
558 bool LoopRotate::simplifyLoopLatch(
Loop *L) {
579 << LastExit->
getName() <<
"\n");
585 unsigned FallThruPath = BI->
getSuccessor(0) == Latch ? 0 : 1;
595 assert(Latch->
empty() &&
"unable to evacuate Latch");
596 LI->removeBlock(Latch);
598 DT->eraseNode(Latch);
604 bool LoopRotate::processLoop(
Loop *L) {
611 bool SimplifiedLatch = simplifyLoopLatch(L);
613 bool MadeChange = rotateLoop(L, SimplifiedLatch);
615 "Loop latch should be exiting after loop-rotate.");
619 if ((MadeChange || SimplifiedLatch) && LoopMD)
626 : EnableHeaderDuplication(EnableHeaderDuplication) {}
632 LoopRotate LR(Threshold, &AR.
LI, &AR.
TTI, &AR.
AC, &AR.
DT, &AR.
SE);
634 bool Changed = LR.processLoop(&L);
642 class LoopRotateLegacyPass :
public LoopPass {
643 unsigned MaxHeaderSize;
647 LoopRotateLegacyPass(
int SpecifiedMaxHeaderSize = -1) :
LoopPass(
ID) {
649 if (SpecifiedMaxHeaderSize == -1)
652 MaxHeaderSize =
unsigned(SpecifiedMaxHeaderSize);
667 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
668 const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
669 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
670 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
671 auto *DT = DTWP ? &DTWP->getDomTree() :
nullptr;
672 auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
673 auto *SE = SEWP ? &SEWP->getSE() :
nullptr;
674 LoopRotate LR(MaxHeaderSize, LI, TTI, AC, DT, SE);
675 return LR.processLoop(L);
690 return new LoopRotateLegacyPass(MaxHeaderSize);
Pass interface - Implemented by all 'passes'.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
use_iterator_impl< Use > use_iterator
A parsed version of the target data layout string in and methods for querying it. ...
Helper class for SSA formation on a set of values defined in multiple blocks.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PreservedAnalyses getLoopPassPreservedAnalyses()
Returns the minimum set of Analyses that all loop passes must preserve.
STATISTIC(NumFunctions,"Total number of functions")
This header provides classes for managing a pipeline of passes over loops in LLVM IR...
This is the interface for a simple mod/ref and alias analysis over globals.
bool convergent
True if this function contains a call to a convergent function.
void Initialize(Type *Ty, StringRef Name)
Reset this object to get ready for a new set of SSA updates with type 'Ty'.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
void AddAvailableValue(BasicBlock *BB, Value *V)
Indicate that a rewritten value is available in the specified block with the specified value...
The main scalar evolution driver.
An immutable pass that tracks lazily created AssumptionCache objects.
bool mayHaveSideEffects() const
Return true if the instruction may have side effects.
A cache of .assume calls within a function.
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop...
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
bool notDuplicatable
True if this function cannot be duplicated.
const std::vector< BlockT * > & getBlocks() const
Get a list of the basic blocks which make up this loop.
BlockT * getHeader() const
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
iterator begin()
Instruction iterator methods.
DomTreeNodeBase< NodeT > * getIDom() const
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
bool isUnconditional() const
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
This is the interface for a SCEV-based alias analysis.
Option class for critical edge splitting.
A Use represents the edge between a Value definition and its users.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
void setName(const Twine &Name)
Change the name of the value.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U)
bool mayReadFromMemory() const
Return true if this instruction may read memory.
BasicBlock * getSuccessor(unsigned i) const
Base class for the actual dominator tree node.
static bool shouldSpeculateInstrs(BasicBlock::iterator Begin, BasicBlock::iterator End, Loop *L)
Determine whether the instructions in this range may be safely and cheaply speculated.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void replaceSuccessorsPhiUsesWith(BasicBlock *New)
Update all phi nodes in this basic block's successors to refer to basic block New instead of to it...
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
If this edge is a critical edge, insert a new node to split the critical edge.
initializer< Ty > init(const Ty &Val)
Value * GetValueInMiddleOfBlock(BasicBlock *BB)
Construct SSA form, materializing a value that is live in the middle of the specified block...
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
LLVM Basic Block Representation.
This is an important class for using LLVM in a threaded context.
static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader, BasicBlock *OrigPreheader, ValueToValueMapTy &ValueMap)
RewriteUsesOfClonedInstructions - We just cloned the instructions from the old header into the prehea...
Conditional or Unconditional Branch instruction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
INITIALIZE_PASS_BEGIN(LoopRotateLegacyPass,"loop-rotate","Rotate Loops", false, false) INITIALIZE_PASS_END(LoopRotateLegacyPass
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
Represent the analysis usage information of a pass.
void splice(iterator where, iplist_impl &L2)
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
const InstListType & getInstList() const
Return the underlying instruction list container.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
User * getUser() const
Returns the User that contains this Use.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
Interval::pred_iterator pred_end(Interval *I)
self_iterator getIterator()
bool HasValueForBlock(BasicBlock *BB) const
Return true if the SSAUpdater already has a value for the specified block.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void initializeLoopRotateLegacyPassPass(PassRegistry &)
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
TargetTransformInfo & TTI
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
Iterator for intrusive lists based on ilist_node.
machine trace Machine Trace Metrics
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
This is the common base class for debug info intrinsics.
Utility to calculate the size and a few similar metrics for a set of basic blocks.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM...
void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
If this flag is set, the remapper ignores missing function-local entries (Argument, Instruction, BasicBlock) that are not in the value map.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
bool MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemoryDependenceResults *MemDep=nullptr)
Attempts to merge a block into its predecessor, if possible.
void moveToHeader(BlockT *BB)
This method is used to move BB (which must be part of this loop) to be the loop header of the loop (t...
Represents a single loop in the control flow graph.
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
void getLoopAnalysisUsage(AnalysisUsage &AU)
Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
LoopRotatePass(bool EnableHeaderDuplication=true)
void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &EphValues)
Add information about a block to the current state.
static int const Threshold
TODO: Write a new FunctionPass AliasAnalysis so that it can keep a cache.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
LLVM Value Representation.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
This is the interface for LLVM's primary stateless and local alias analysis.
A container for analyses that lazily runs them and caches their results.
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
See if we can compute a simplified version of this instruction.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop)...
unsigned NumInsts
Number of instructions in the analyzed blocks.
void RewriteUse(Use &U)
Rewrite a use of the symbolic value.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const BasicBlock * getParent() const
Pass * createLoopRotatePass(int MaxHeaderSize=-1)
static cl::opt< unsigned > DefaultRotationThreshold("rotation-max-header-size", cl::init(16), cl::Hidden, cl::desc("The default maximum header size for automatic loop rotation"))