52 #define DEBUG_TYPE "machine-sink"
56 cl::desc(
"Split critical edges during machine sinking"),
61 cl::desc(
"Use block frequency info to find successors to sink"),
65 "machine-sink-split-probability-threshold",
67 "Percentage threshold for splitting single-instruction critical edge. "
68 "If the branch threshold is higher than this threshold, we allow "
69 "speculative execution of up to 1 instruction to avoid branching to "
70 "splitted critical edge"),
73 STATISTIC(NumSunk,
"Number of machine instructions sunk");
74 STATISTIC(NumSplit,
"Number of critical edges split");
75 STATISTIC(NumCoalesces,
"Number of copies coalesced");
100 typedef std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>
127 void releaseMemory()
override {
128 CEBCandidates.clear();
153 AllSuccsCache &AllSuccessors);
156 bool &BreakPHIEdge,
bool &LocalUse)
const;
158 bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
162 AllSuccsCache &AllSuccessors);
169 AllSuccsCache &AllSuccessors)
const;
177 "Machine code sinking",
false,
false)
185 bool MachineSinking::PerformTrivialForwardCoalescing(
MachineInstr &
MI,
190 unsigned SrcReg = MI.getOperand(1).getReg();
191 unsigned DstReg = MI.getOperand(0).getReg();
194 !
MRI->hasOneNonDBGUse(SrcReg))
205 DEBUG(
dbgs() <<
"Coalescing: " << *DefMI);
207 MRI->replaceRegWith(DstReg, SrcReg);
208 MI.eraseFromParent();
212 MRI->clearKillFlags(SrcReg);
227 bool &LocalUse)
const {
229 "Only makes sense for vregs");
232 if (
MRI->use_nodbg_empty(Reg))
253 unsigned OpNo = &MO - &UseInst->
getOperand(0);
255 if (!(UseBlock == MBB && UseInst->
isPHI() &&
257 BreakPHIEdge =
false;
267 unsigned OpNo = &MO - &UseInst->
getOperand(0);
269 if (UseInst->
isPHI()) {
273 }
else if (UseBlock == DefMBB) {
279 if (!DT->dominates(MBB, UseBlock))
290 DEBUG(
dbgs() <<
"******** Machine Sinking ********\n");
295 DT = &getAnalysis<MachineDominatorTree>();
296 PDT = &getAnalysis<MachinePostDominatorTree>();
297 LI = &getAnalysis<MachineLoopInfo>();
298 MBFI =
UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() :
nullptr;
299 MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
300 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
302 bool EverMadeChange =
false;
305 bool MadeChange =
false;
308 CEBCandidates.clear();
314 for (
auto &Pair : ToSplit) {
315 auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *
this);
316 if (NewSucc !=
nullptr) {
317 DEBUG(
dbgs() <<
" *** Splitting critical edge:"
318 " BB#" << Pair.first->getNumber()
319 <<
" -- BB#" << NewSucc->getNumber()
320 <<
" -- BB#" << Pair.second->getNumber() <<
'\n');
324 DEBUG(
dbgs() <<
" *** Not legal to break critical edge\n");
327 if (!MadeChange)
break;
328 EverMadeChange =
true;
332 for (
auto I : RegsToClearKillFlags)
333 MRI->clearKillFlags(
I);
334 RegsToClearKillFlags.clear();
336 return EverMadeChange;
346 if (!DT->isReachableFromEntry(&MBB))
return false;
348 bool MadeChange =
false;
351 AllSuccsCache AllSuccessors;
356 bool ProcessedBegin,
SawStore =
false;
362 ProcessedBegin = I == MBB.
begin();
369 bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
381 }
while (!ProcessedBegin);
386 bool MachineSinking::isWorthBreakingCriticalEdge(
MachineInstr &MI,
394 if (!CEBCandidates.insert(std::make_pair(From, To)).second)
397 if (!MI.
isCopy() && !
TII->isAsCheapAsAMove(MI))
400 if (From->
isSuccessor(To) && MBPI->getEdgeProbability(From, To) <=
411 unsigned Reg = MO.
getReg();
423 if (
MRI->hasOneNonDBGUse(Reg)) {
437 bool MachineSinking::PostponeSplitCriticalEdge(
MachineInstr &MI,
441 if (!isWorthBreakingCriticalEdge(MI, FromBB, ToBB))
449 if (LI->getLoopFor(FromBB) == LI->getLoopFor(ToBB) &&
450 LI->isLoopHeader(ToBB))
497 if (!DT->dominates(ToBB, *PI))
502 ToSplit.insert(std::make_pair(FromBB, ToBB));
518 if (!DI->isDebugValue())
520 if (DI->getOperand(0).isReg() &&
527 bool MachineSinking::isProfitableToSinkTo(
unsigned Reg,
MachineInstr &MI,
530 AllSuccsCache &AllSuccessors) {
531 assert (SuccToSinkTo &&
"Invalid SinkTo Candidate BB");
533 if (MBB == SuccToSinkTo)
537 if (!PDT->dominates(SuccToSinkTo, MBB))
542 if (LI->getLoopDepth(MBB) > LI->getLoopDepth(SuccToSinkTo))
546 bool NonPHIUse =
false;
549 if (UseBlock == SuccToSinkTo && !UseInst.
isPHI())
557 bool BreakPHIEdge =
false;
560 FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
561 return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
572 AllSuccsCache &AllSuccessors)
const {
575 auto Succs = AllSuccessors.find(MBB);
576 if (Succs != AllSuccessors.end())
577 return Succs->second;
589 const std::vector<MachineDomTreeNode *> &Children =
590 DT->getNode(MBB)->getChildren();
591 for (
const auto &DTChild : Children)
593 if (DTChild->getIDom()->getBlock() == MI.
getParent() &&
600 AllSuccs.begin(), AllSuccs.end(),
602 uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
603 uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
604 bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
605 return HasBlockFreq ? LHSFreq < RHSFreq
606 : LI->getLoopDepth(L) < LI->getLoopDepth(R);
609 auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs));
611 return it.first->second;
618 AllSuccsCache &AllSuccessors) {
619 assert (MBB &&
"Invalid MachineBasicBlock!");
629 if (!MO.
isReg())
continue;
631 unsigned Reg = MO.
getReg();
632 if (Reg == 0)
continue;
639 if (!
MRI->isConstantPhysReg(Reg))
641 }
else if (!MO.
isDead()) {
647 if (MO.
isUse())
continue;
650 if (!
TII->isSafeToMoveRegClassDefs(
MRI->getRegClass(Reg)))
658 bool LocalUse =
false;
660 BreakPHIEdge, LocalUse))
671 GetAllSortedSuccessors(MI, MBB, AllSuccessors)) {
672 bool LocalUse =
false;
674 BreakPHIEdge, LocalUse)) {
675 SuccToSinkTo = SuccBlock;
686 if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo, AllSuccessors))
693 if (MBB == SuccToSinkTo)
698 if (SuccToSinkTo && SuccToSinkTo->
isEHPad())
721 auto *PredBB = PredMBB->getBasicBlock();
738 MachineBranchPredicate MBP;
742 return MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
745 MBP.LHS.getReg() == BaseReg;
751 AllSuccsCache &AllSuccessors) {
753 if (!
TII->shouldSink(MI))
778 bool BreakPHIEdge =
false;
781 FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge, AllSuccessors);
793 if (!MO.
isReg())
continue;
794 unsigned Reg = MO.
getReg();
800 DEBUG(
dbgs() <<
"Sink instr " << MI <<
"\tinto block " << *SuccToSinkTo);
807 bool TryBreak =
false;
810 DEBUG(
dbgs() <<
" *** NOTE: Won't sink load along critical edge.\n");
816 if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
817 DEBUG(
dbgs() <<
" *** NOTE: Critical edge found\n");
822 if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
823 DEBUG(
dbgs() <<
" *** NOTE: Loop header found\n");
829 DEBUG(
dbgs() <<
"Sinking along critical edge.\n");
835 PostponeSplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
837 DEBUG(
dbgs() <<
" *** PUNTING: Not legal or profitable to "
838 "break critical edge\n");
848 bool Status = PostponeSplitCriticalEdge(MI, ParentBlock,
849 SuccToSinkTo, BreakPHIEdge);
851 DEBUG(
dbgs() <<
" *** PUNTING: Not legal or profitable to "
852 "break critical edge\n");
859 while (InsertPos != SuccToSinkTo->
end() && InsertPos->isPHI())
867 SuccToSinkTo->
splice(InsertPos, ParentBlock, MI,
872 DBE = DbgValuesToSink.
end(); DBI != DBE; ++DBI) {
874 SuccToSinkTo->
splice(InsertPos, ParentBlock, DbgMI,
885 RegsToClearKillFlags.set(MO.
getReg());
unsigned succ_size() const
void push_back(const T &Elt)
bool isEHPad() const
Returns true if the block is a landing pad.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static bool AllUsesDominatedByBlock(Instruction *Inst, BasicBlock *BB, DominatorTree &DT)
AllUsesDominatedByBlock - Return true if all uses of the specified value occur in blocks dominated by...
STATISTIC(NumFunctions,"Total number of functions")
MachineBasicBlock * getMBB() const
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
virtual bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify=false) const
Analyze the branching code at the end of MBB and parse it into the MachineBranchPredicate structure i...
bool isPredicable(QueryType Type=AllInBundle) const
Return true if this instruction has a predicate operand that controls execution.
machine Machine code sinking
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
iterator_range< mop_iterator > operands()
Represents a predicate at the MachineFunction level.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
bool isCopyLike() const
Return true if the instruction behaves like a copy.
std::vector< MachineBasicBlock * >::iterator pred_iterator
COFF::MachineTypes Machine
static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
bool isConvergent(QueryType Type=AnyInBundle) const
Return true if this instruction is convergent.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
initializer< Ty > init(const Ty &Val)
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
succ_iterator succ_begin()
static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
Return true if MI is likely to be usable as a memory operation by the implicit null check optimizatio...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
pred_iterator pred_begin()
machine Machine code false
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
static bool SinkInstruction(Instruction *Inst, SmallPtrSetImpl< Instruction * > &Stores, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
SinkInstruction - Determine whether it is safe to sink the specified machine instruction out of its c...
static cl::opt< unsigned > SplitEdgeProbabilityThreshold("machine-sink-split-probability-threshold", cl::desc("Percentage threshold for splitting single-instruction critical edge. ""If the branch threshold is higher than this threshold, we allow ""speculative execution of up to 1 instruction to avoid branching to ""splitted critical edge"), cl::init(40), cl::Hidden)
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
virtual bool getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, int64_t &Offset, const TargetRegisterInfo *TRI) const
Get the base register and byte offset of an instruction that reads/writes memory. ...
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void initializeMachineSinkingPass(PassRegistry &)
INITIALIZE_PASS_BEGIN(MachineSinking,"machine-sink","Machine code sinking", false, false) INITIALIZE_PASS_END(MachineSinking
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
static void collectDebugValues(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DbgValues)
collectDebgValues - Scan instructions following MI and collect any matching DBG_VALUEs.
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
virtual const TargetInstrInfo * getInstrInfo() const
A vector that has set insertion semantics.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
machine sink
When an instruction is found to only be used outside of the loop, this function moves it to the exit ...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const