33 #define DEBUG_TYPE "tailduplication"
35 STATISTIC(NumTails,
"Number of tails duplicated");
36 STATISTIC(NumTailDups,
"Number of tail duplicated blocks");
38 "Number of instructions added due to tail duplication");
40 "Number of instructions removed due to tail duplication");
41 STATISTIC(NumDeadBlocks,
"Number of dead blocks removed");
42 STATISTIC(NumAddedPHIs,
"Number of phis added");
53 "tail-dup-indirect-size",
54 cl::desc(
"Maximum instructions to consider tail duplicating blocks that "
55 "end with indirect branches."),
cl::init(20),
60 cl::desc(
"Verify sanity of PHI instructions during taildup"),
68 bool LayoutModeIn,
unsigned TailDupSizeIn) {
75 TailDupSize = TailDupSizeIn;
77 assert(MBPI !=
nullptr &&
"Machine Branch Probability Info required");
79 LayoutMode = LayoutModeIn;
80 PreRegAlloc = MRI->
isSSA();
89 while (MI != MBB->
end()) {
94 for (
unsigned i = 1, e = MI->getNumOperands();
i != e;
i += 2) {
96 if (PHIBB == PredBB) {
103 dbgs() <<
" missing input from predecessor BB#"
104 << PredBB->getNumber() <<
'\n';
109 for (
unsigned i = 1, e = MI->getNumOperands();
i != e;
i += 2) {
111 if (CheckExtra && !Preds.count(PHIBB)) {
112 dbgs() <<
"Warning: malformed PHI in BB#" << MBB->
getNumber() <<
": "
114 dbgs() <<
" extra input from predecessor BB#" << PHIBB->
getNumber()
148 if (!tailDuplicate(IsSimple, MBB, ForcedLayoutPred, TDBBs, Copies))
161 updateSuccessorsPHIs(MBB, isDead, TDBBs, Succs);
165 NumTailDupRemoved += MBB->
size();
166 removeDeadBlock(MBB, RemovalCallback);
171 if (!SSAUpdateVRs.
empty()) {
172 for (
unsigned i = 0, e = SSAUpdateVRs.
size();
i != e; ++
i) {
173 unsigned VReg = SSAUpdateVRs[
i];
187 SSAUpdateVals.
find(VReg);
188 for (
unsigned j = 0, ee = LI->second.
size(); j != ee; ++j) {
190 unsigned SrcReg = LI->second[j].second;
214 SSAUpdateVRs.
clear();
215 SSAUpdateVals.
clear();
220 for (
unsigned i = 0, e = Copies.
size();
i != e; ++
i) {
235 NumAddedPHIs += NewPHIs.
size();
238 *DuplicatedPreds = std::move(TDBBs);
247 bool MadeChange =
false;
250 DEBUG(
dbgs() <<
"\n*** Before tail-duplicating\n");
277 if (
UseMI.isDebugValue())
279 if (
UseMI.getParent() != BB)
297 for (
const auto &
MI : BB) {
300 for (
unsigned i = 1, e =
MI.getNumOperands();
i != e;
i += 2) {
301 unsigned SrcReg =
MI.getOperand(
i).getReg();
302 UsedByPhi->
insert(SrcReg);
308 void TailDuplicator::addSSAUpdateEntry(
unsigned OrigReg,
unsigned NewReg,
311 SSAUpdateVals.
find(OrigReg);
312 if (LI != SSAUpdateVals.
end())
313 LI->second.push_back(std::make_pair(BB, NewReg));
316 Vals.push_back(std::make_pair(BB, NewReg));
317 SSAUpdateVals.
insert(std::make_pair(OrigReg, Vals));
324 void TailDuplicator::processPHI(
331 assert(SrcOpIdx &&
"Unable to find matching PHI source?");
335 LocalVRMap.
insert(std::make_pair(DefReg, RegSubRegPair(SrcReg, SrcSubReg)));
340 Copies.push_back(std::make_pair(NewDef, RegSubRegPair(SrcReg, SrcSubReg)));
342 addSSAUpdateEntry(DefReg, NewDef, PredBB);
356 void TailDuplicator::duplicateInstruction(
373 LocalVRMap.
insert(std::make_pair(Reg, RegSubRegPair(NewReg, 0)));
375 addSSAUpdateEntry(Reg, NewReg, PredBB);
377 auto VI = LocalVRMap.
find(Reg);
378 if (
VI != LocalVRMap.
end()) {
385 if (
VI->second.SubReg != 0) {
413 if (NewRC ==
nullptr)
417 TII->
get(TargetOpcode::COPY), NewReg)
418 .addReg(
VI->second.Reg, 0,
VI->second.SubReg);
420 LocalVRMap.
insert(std::make_pair(Reg, RegSubRegPair(NewReg, 0)));
440 void TailDuplicator::updateSuccessorsPHIs(
452 if (MO.
getMBB() == FromBB) {
460 unsigned Reg = MO0.
getReg();
467 if (MO.
getMBB() == FromBB) {
479 SSAUpdateVals.
find(Reg);
480 if (LI != SSAUpdateVals.
end()) {
482 for (
unsigned j = 0, ee = LI->second.
size(); j != ee; ++j) {
491 unsigned SrcReg = LI->second[j].second;
497 MIB.addReg(SrcReg).addMBB(SrcBB);
502 for (
unsigned j = 0, ee = TDBBs.
size(); j != ee; ++j) {
509 MIB.addReg(Reg).addMBB(SrcBB);
537 unsigned MaxDuplicateCount;
538 if (TailDupSize == 0 &&
541 MaxDuplicateCount = 1;
542 else if (TailDupSize == 0)
545 MaxDuplicateCount = TailDupSize;
553 if (TII->
analyzeBranch(TailBB, PredTBB, PredFBB, PredCond) &&
563 bool HasIndirectbr =
false;
567 if (HasIndirectbr && PreRegAlloc)
592 if (PreRegAlloc && MI.
isCall())
598 if (InstrCount > MaxDuplicateCount)
611 for (
auto SB : TailBB.successors()) {
612 for (
auto &
I : *
SB) {
623 if (HasIndirectbr && PreRegAlloc)
632 return canCompletelyDuplicateBB(TailBB);
642 if (I == TailBB->
end())
644 return I->isUnconditionalBranch();
666 if (!PredCond.
empty())
672 bool TailDuplicator::duplicateSimpleBB(
680 bool Changed =
false;
694 DEBUG(
dbgs() <<
"\nTail-duplicating into PredBB: " << *PredBB
695 <<
"From simple Succ: " << *TailBB);
701 if (PredCond.
empty())
711 if (PredFBB == TailBB)
713 if (PredTBB == TailBB)
717 if (PredTBB == PredFBB) {
723 if (PredFBB == NextBB)
725 if (PredTBB == NextBB && PredFBB ==
nullptr)
755 if (!PredCond.
empty())
780 return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies);
785 bool Changed =
false;
789 assert(TailBB != PredBB &&
790 "Single-block loop should have been rejected earlier!");
796 bool IsLayoutSuccessor =
false;
797 if (ForcedLayoutPred)
798 IsLayoutSuccessor = (ForcedLayoutPred == PredBB);
800 IsLayoutSuccessor =
true;
801 if (IsLayoutSuccessor)
804 DEBUG(
dbgs() <<
"\nTail-duplicating into PredBB: " << *PredBB
805 <<
"From Succ: " << *TailBB);
824 processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi,
true);
828 duplicateInstruction(MI, TailBB, PredBB, LocalVRMap, UsedByPhi);
831 appendCopies(PredBB, CopyInfos, Copies);
838 NumTailDupAdded += TailBB->
size() - 1;
843 "TailDuplicate called on block with multiple successors!");
864 !TII->
analyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond) &&
866 (!PriorTBB || PriorTBB == TailBB) &&
869 DEBUG(
dbgs() <<
"\nMerging into block: " << *PrevBB
870 <<
"From MBB: " << *TailBB);
880 while (I != TailBB->
end() && I->isPHI()) {
884 processPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi,
true);
888 while (I != TailBB->
end()) {
892 assert(!MI->
isBundle() &&
"Not expecting bundles before regalloc!");
893 duplicateInstruction(MI, TailBB, PrevBB, LocalVRMap, UsedByPhi);
896 appendCopies(PrevBB, CopyInfos, Copies);
943 while (I != TailBB->
end() && I->isPHI()) {
947 processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi,
false);
949 appendCopies(PredBB, CopyInfos, Copies);
962 for (
auto &CI : CopyInfos) {
964 .
addReg(CI.second.Reg, 0, CI.second.SubReg);
971 void TailDuplicator::removeDeadBlock(
975 DEBUG(
dbgs() <<
"\nRemoving MBB: " << *MBB);
978 (*RemovalCallback)(
MBB);
unsigned succ_size() const
void push_back(const T &Elt)
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
virtual MachineInstr * duplicate(MachineInstr &Orig, MachineFunction &MF) const
Create a duplicate of the Orig instruction in MF.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
instr_iterator instr_begin()
instr_iterator instr_end()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
STATISTIC(NumFunctions,"Total number of functions")
MachineBasicBlock * getMBB() const
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Implements a dense probed hash-table based set.
iterator getFirstNonDebugInstr()
Returns an iterator to the first non-debug instruction in the basic block, or end().
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Describe properties that are true of each instruction in the target description file.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static cl::opt< bool > TailDupVerify("tail-dup-verify", cl::desc("Verify sanity of PHI instructions during taildup"), cl::init(false), cl::Hidden)
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
static void VerifyPHIs(MachineFunction &MF, bool CheckExtra)
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
An efficient, type-erasing, non-owning reference to a callable.
virtual bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
void RewriteUse(MachineOperand &U)
RewriteUse - Rewrite a use of the symbolic value.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
static unsigned InstrCount
static bool isSimpleBB(MachineBasicBlock *TailBB)
True if this BB has only one unconditional jump.
bool optForSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
iterator_range< succ_iterator > successors()
static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, const MachineRegisterInfo *MRI)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
static use_iterator use_end()
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
Reg
All possible values of the reg field in the ModR/M byte.
LLVM_NODISCARD bool empty() const
unsigned getNumOperands() const
Access to explicit operands of the instruction.
void initMF(MachineFunction &MF, const MachineBranchProbabilityInfo *MBPI, bool LayoutMode, unsigned TailDupSize=0)
Prepare to run on a specific machine function.
void RemoveOperand(unsigned i)
Erase an operand from an instruction, leaving it with one fewer operand than it started with...
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB)
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
virtual const TargetRegisterClass * getMatchingSuperRegClass(const TargetRegisterClass *A, const TargetRegisterClass *B, unsigned Idx) const
Return a subclass of the specified register class A so that each register in it has a sub-register of...
void Initialize(unsigned V)
Initialize - Reset this object to get ready for a new set of SSA updates.
bool isConvergent(QueryType Type=AnyInBundle) const
Return true if this instruction is convergent.
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
bool isDebugValue() const
INITIALIZE_PASS(HexagonEarlyIfConversion,"hexagon-eif","Hexagon early if conversion", false, false) bool HexagonEarlyIfConversion MachineBasicBlock * SB
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
static cl::opt< unsigned > TailDuplicateSize("tail-dup-size", cl::desc("Maximum instructions to consider tail duplicating"), cl::init(2), cl::Hidden)
initializer< Ty > init(const Ty &Val)
bool erase(const KeyT &Val)
bool isReturn(QueryType Type=AnyInBundle) const
unsigned const MachineRegisterInfo * MRI
virtual unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const
Remove the branching code at the end of the specific MBB.
MachineInstrBuilder & UseMI
bool shouldTailDuplicate(bool IsSimple, MachineBasicBlock &TailBB)
Determine if it is profitable to duplicate this block.
const MachineOperand & getOperand(unsigned i) const
std::pair< iterator, bool > insert(const ValueT &V)
bool isIndirectBranch(QueryType Type=AnyInBundle) const
Return true if this is an indirect branch, such as a branch through a register.
void setMBB(MachineBasicBlock *MBB)
self_iterator getIterator()
iterator_range< pred_iterator > predecessors()
succ_iterator succ_begin()
unsigned getSubReg() const
iterator_range< use_instr_iterator > use_instructions(unsigned Reg) const
bool isNotDuplicable(QueryType Type=AnyInBundle) const
Return true if this instruction cannot be safely duplicated.
pred_iterator pred_begin()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
void setIsKill(bool Val=true)
static void getRegsUsedByPHIs(const MachineBasicBlock &BB, DenseSet< unsigned > *UsedByPhi)
bool canTailDuplicate(MachineBasicBlock *TailBB, MachineBasicBlock *PredBB)
Returns true if TailBB can successfully be duplicated into PredBB.
A SetVector that performs no allocations if smaller than a certain size.
bool hasEHPadSuccessor() const
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
bool tailDuplicateAndUpdate(bool IsSimple, MachineBasicBlock *MBB, MachineBasicBlock *ForcedLayoutPred, SmallVectorImpl< MachineBasicBlock * > *DuplicatedPreds=nullptr, llvm::function_ref< void(MachineBasicBlock *)> *RemovalCallback=nullptr)
Tail duplicate a single basic block into its predecessors, and then clean up.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static bool bothUsedInPHI(const MachineBasicBlock &A, const SmallPtrSet< MachineBasicBlock *, 8 > &SuccsB)
cl::opt< unsigned > TailDupIndirectBranchSize
static cl::opt< unsigned > TailDupLimit("tail-dup-limit", cl::init(~0U), cl::Hidden)
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
const TargetRegisterClass * getRegClassConstraint(unsigned OpIdx, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
Compute the static register class constraint for operand OpIdx.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
bool hasAddressTaken() const
Test whether this block is potentially the target of an indirect branch.
size_type count(const ValueT &V) const
Return 1 if the specified key is in the set, 0 otherwise.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
use_iterator use_begin(unsigned RegNo) const
bool hasOneNonDBGUse(unsigned RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug instruction using the specified regis...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
bool tailDuplicateBlocks()
Look for small blocks that are unconditionally branched to and do not fall through.
bool isCall(QueryType Type=AnyInBundle) const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
void setSubReg(unsigned subReg)
iterator find(const KeyT &Val)
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void AddAvailableValue(MachineBasicBlock *BB, unsigned V)
AddAvailableValue - Indicate that a rewritten value is available at the end of the specified block wi...
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
void setRegClass(unsigned Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
MachineModuleInfo & getMMI() const
unsigned composeSubRegIndices(unsigned a, unsigned b) const
Return the subregister index you get from composing two subregister indices.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
unsigned pred_size() const
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.