38 #define DEBUG_TYPE "machine-sink"
42 cl::desc(
"Split critical edges during machine sinking"),
47 cl::desc(
"Use block frequency info to find successors to sink"),
51 STATISTIC(NumSunk,
"Number of machine instructions sunk");
52 STATISTIC(NumSplit,
"Number of critical edges split");
53 STATISTIC(NumCoalesces,
"Number of copies coalesced");
76 typedef std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>
101 void releaseMemory()
override {
102 CEBCandidates.clear();
127 AllSuccsCache &AllSuccessors);
130 bool &BreakPHIEdge,
bool &LocalUse)
const;
132 bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
136 AllSuccsCache &AllSuccessors);
143 AllSuccsCache &AllSuccessors)
const;
150 "Machine code sinking",
false,
false)
157 bool MachineSinking::PerformTrivialForwardCoalescing(
MachineInstr *
MI,
162 unsigned SrcReg = MI->getOperand(1).getReg();
163 unsigned DstReg = MI->getOperand(0).getReg();
166 !MRI->hasOneNonDBGUse(SrcReg))
177 DEBUG(
dbgs() <<
"Coalescing: " << *DefMI);
179 MRI->replaceRegWith(DstReg, SrcReg);
180 MI->eraseFromParent();
184 MRI->clearKillFlags(SrcReg);
195 MachineSinking::AllUsesDominatedByBlock(
unsigned Reg,
199 bool &LocalUse)
const {
201 "Only makes sense for vregs");
204 if (MRI->use_nodbg_empty(Reg))
225 unsigned OpNo = &MO - &UseInst->
getOperand(0);
227 if (!(UseBlock == MBB && UseInst->
isPHI() &&
229 BreakPHIEdge =
false;
239 unsigned OpNo = &MO - &UseInst->
getOperand(0);
241 if (UseInst->
isPHI()) {
245 }
else if (UseBlock == DefMBB) {
251 if (!DT->dominates(MBB, UseBlock))
262 DEBUG(
dbgs() <<
"******** Machine Sinking ********\n");
267 DT = &getAnalysis<MachineDominatorTree>();
268 PDT = &getAnalysis<MachinePostDominatorTree>();
269 LI = &getAnalysis<MachineLoopInfo>();
270 MBFI =
UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() :
nullptr;
271 AA = &getAnalysis<AliasAnalysis>();
273 bool EverMadeChange =
false;
276 bool MadeChange =
false;
279 CEBCandidates.clear();
282 MadeChange |= ProcessBlock(MBB);
285 for (
auto &Pair : ToSplit) {
286 auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second,
this);
287 if (NewSucc !=
nullptr) {
288 DEBUG(
dbgs() <<
" *** Splitting critical edge:"
289 " BB#" << Pair.first->getNumber()
290 <<
" -- BB#" << NewSucc->getNumber()
291 <<
" -- BB#" << Pair.second->getNumber() <<
'\n');
295 DEBUG(
dbgs() <<
" *** Not legal to break critical edge\n");
298 if (!MadeChange)
break;
299 EverMadeChange =
true;
303 for (
auto I : RegsToClearKillFlags)
304 MRI->clearKillFlags(
I);
305 RegsToClearKillFlags.clear();
307 return EverMadeChange;
317 if (!DT->isReachableFromEntry(&MBB))
return false;
319 bool MadeChange =
false;
322 AllSuccsCache AllSuccessors;
327 bool ProcessedBegin,
SawStore =
false;
333 ProcessedBegin = I == MBB.
begin();
340 bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
346 if (SinkInstruction(MI, SawStore, AllSuccessors))
347 ++NumSunk, MadeChange =
true;
350 }
while (!ProcessedBegin);
355 bool MachineSinking::isWorthBreakingCriticalEdge(
MachineInstr *MI,
363 if (!CEBCandidates.insert(std::make_pair(From, To)).second)
366 if (!MI->
isCopy() && !
TII->isAsCheapAsAMove(MI))
376 unsigned Reg = MO.
getReg();
388 if (MRI->hasOneNonDBGUse(Reg)) {
402 bool MachineSinking::PostponeSplitCriticalEdge(
MachineInstr *MI,
406 if (!isWorthBreakingCriticalEdge(MI, FromBB, ToBB))
414 if (LI->getLoopFor(FromBB) == LI->getLoopFor(ToBB) &&
415 LI->isLoopHeader(ToBB))
459 E = ToBB->
pred_end(); PI != E; ++PI) {
462 if (!DT->dominates(ToBB, *PI))
467 ToSplit.insert(std::make_pair(FromBB, ToBB));
487 if (!DI->isDebugValue())
489 if (DI->getOperand(0).isReg() &&
496 bool MachineSinking::isProfitableToSinkTo(
unsigned Reg,
MachineInstr *MI,
499 AllSuccsCache &AllSuccessors) {
500 assert (MI &&
"Invalid MachineInstr!");
501 assert (SuccToSinkTo &&
"Invalid SinkTo Candidate BB");
503 if (MBB == SuccToSinkTo)
507 if (!PDT->dominates(SuccToSinkTo, MBB))
512 if (LI->getLoopDepth(MBB) > LI->getLoopDepth(SuccToSinkTo))
516 bool NonPHIUse =
false;
517 for (
MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg)) {
519 if (UseBlock == SuccToSinkTo && !UseInst.
isPHI())
527 bool BreakPHIEdge =
false;
530 FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
531 return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
542 AllSuccsCache &AllSuccessors)
const {
545 auto Succs = AllSuccessors.find(MBB);
546 if (Succs != AllSuccessors.end())
547 return Succs->second;
559 const std::vector<MachineDomTreeNode *> &Children =
560 DT->getNode(MBB)->getChildren();
561 for (
const auto &DTChild : Children)
563 if (DTChild->getIDom()->getBlock() == MI->
getParent() &&
570 AllSuccs.begin(), AllSuccs.end(),
572 uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
573 uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
574 bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
575 return HasBlockFreq ? LHSFreq < RHSFreq
576 : LI->getLoopDepth(L) < LI->getLoopDepth(R);
579 auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs));
581 return it.first->second;
588 AllSuccsCache &AllSuccessors) {
590 assert (MI &&
"Invalid MachineInstr!");
591 assert (MBB &&
"Invalid MachineBasicBlock!");
601 if (!MO.
isReg())
continue;
603 unsigned Reg = MO.
getReg();
604 if (Reg == 0)
continue;
611 if (!MRI->isConstantPhysReg(Reg, *MBB->
getParent()))
613 }
else if (!MO.
isDead()) {
619 if (MO.
isUse())
continue;
622 if (!
TII->isSafeToMoveRegClassDefs(MRI->getRegClass(Reg)))
630 bool LocalUse =
false;
631 if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, MBB,
632 BreakPHIEdge, LocalUse))
643 GetAllSortedSuccessors(MI, MBB, AllSuccessors)) {
644 bool LocalUse =
false;
645 if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB,
646 BreakPHIEdge, LocalUse)) {
647 SuccToSinkTo = SuccBlock;
658 if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo, AllSuccessors))
665 if (MBB == SuccToSinkTo)
678 bool MachineSinking::SinkInstruction(
MachineInstr *MI,
bool &SawStore,
679 AllSuccsCache &AllSuccessors) {
701 bool BreakPHIEdge =
false;
704 FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge, AllSuccessors);
716 if (!MO.
isReg())
continue;
717 unsigned Reg = MO.
getReg();
723 DEBUG(
dbgs() <<
"Sink instr " << *MI <<
"\tinto block " << *SuccToSinkTo);
730 bool TryBreak =
false;
733 DEBUG(
dbgs() <<
" *** NOTE: Won't sink load along critical edge.\n");
739 if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
740 DEBUG(
dbgs() <<
" *** NOTE: Critical edge found\n");
745 if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
746 DEBUG(
dbgs() <<
" *** NOTE: Loop header found\n");
752 DEBUG(
dbgs() <<
"Sinking along critical edge.\n");
758 PostponeSplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
760 DEBUG(
dbgs() <<
" *** PUNTING: Not legal or profitable to "
761 "break critical edge\n");
771 bool Status = PostponeSplitCriticalEdge(MI, ParentBlock,
772 SuccToSinkTo, BreakPHIEdge);
774 DEBUG(
dbgs() <<
" *** PUNTING: Not legal or profitable to "
775 "break critical edge\n");
782 while (InsertPos != SuccToSinkTo->
end() && InsertPos->isPHI())
790 SuccToSinkTo->
splice(InsertPos, ParentBlock, MI,
795 DBE = DbgValuesToSink.
end(); DBI != DBE; ++DBI) {
797 SuccToSinkTo->
splice(InsertPos, ParentBlock, DbgMI,
808 RegsToClearKillFlags.set(MO.
getReg());
unsigned succ_size() const
void push_back(const T &Elt)
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
MachineBasicBlock * getMBB() const
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
machine Machine code sinking
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
iterator_range< mop_iterator > operands()
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
bool isCopyLike() const
Return true if the instruction behaves like a copy.
std::vector< MachineBasicBlock * >::iterator pred_iterator
COFF::MachineTypes Machine
bool isConvergent(QueryType Type=AnyInBundle) const
Return true if this instruction is convergent.
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
bool isInsertSubreg() const
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
static bool AvoidsSinking(MachineInstr *MI, MachineRegisterInfo *MRI)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
Represent the analysis usage information of a pass.
#define INITIALIZE_AG_DEPENDENCY(depName)
succ_iterator succ_begin()
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
pred_iterator pred_begin()
machine Machine code false
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static void collectDebugValues(MachineInstr *MI, SmallVectorImpl< MachineInstr * > &DbgValues)
collectDebgValues - Scan instructions following MI and collect any matching DBG_VALUEs.
bool isSubregToReg() const
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
bool isLandingPad() const
isLandingPad - Returns true if the block is a landing pad.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
bool isLiveIn(unsigned Reg) const
isLiveIn - Return true if the specified register is in the live in set.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void initializeMachineSinkingPass(PassRegistry &)
INITIALIZE_PASS_BEGIN(MachineSinking,"machine-sink","Machine code sinking", false, false) INITIALIZE_PASS_END(MachineSinking
unsigned getReg() const
getReg - Returns the register number.
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
virtual const TargetInstrInfo * getInstrInfo() const
A vector that has set insertion semantics.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool isRegSequence() const
machine sink
When an instruction is found to only be used outside of the loop, this function moves it to the exit ...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const