42 #define DEBUG_TYPE "early-ifcvt"
48 cl::desc(
"Maximum number of instructions per speculated block."));
54 STATISTIC(NumDiamondsSeen,
"Number of diamonds");
55 STATISTIC(NumDiamondsConv,
"Number of diamonds converted");
56 STATISTIC(NumTrianglesSeen,
"Number of triangles");
57 STATISTIC(NumTrianglesConv,
"Number of triangles converted");
101 bool isTriangle()
const {
return TBB == Tail || FBB == Tail; }
114 int CondCycles, TCycles, FCycles;
117 : PHI(phi), TReg(0), FReg(0), CondCycles(0), TCycles(0), FCycles(0) {}
145 bool findInsertionPoint();
148 void replacePHIInstrs();
151 void rewritePHIOperands();
159 LiveRegUnits.clear();
160 LiveRegUnits.setUniverse(TRI->getNumRegUnits());
161 ClobberedRegUnits.clear();
162 ClobberedRegUnits.resize(TRI->getNumRegUnits());
198 if (
I->isDebugValue())
217 DEBUG(
dbgs() <<
"Won't speculate load: " << *
I);
222 bool DontMoveAcrossStore =
true;
223 if (!
I->isSafeToMove(
nullptr, DontMoveAcrossStore)) {
230 if (MO.isRegMask()) {
231 DEBUG(
dbgs() <<
"Won't speculate regmask: " << *
I);
236 unsigned Reg = MO.getReg();
241 ClobberedRegUnits.set(*Units);
246 if (!DefMI || DefMI->
getParent() != Head)
248 if (InsertAfter.insert(DefMI).second)
251 DEBUG(
dbgs() <<
"Can't insert instructions below terminator.\n");
270 bool SSAIfConv::findInsertionPoint() {
273 LiveRegUnits.clear();
281 if (InsertAfter.count(&*I)) {
282 DEBUG(
dbgs() <<
"Can't insert code after " << *I);
291 unsigned Reg = MO.getReg();
297 LiveRegUnits.erase(*Units);
303 while (!Reads.
empty())
306 if (ClobberedRegUnits.test(*Units))
307 LiveRegUnits.insert(*Units);
310 if (I != FirstTerm && I->isTerminator())
315 if (!LiveRegUnits.empty()) {
317 dbgs() <<
"Would clobber";
319 i = LiveRegUnits.begin(), e = LiveRegUnits.end();
i != e; ++
i)
321 dbgs() <<
" live before " << *
I;
328 DEBUG(
dbgs() <<
"Can insert before " << *I);
331 DEBUG(
dbgs() <<
"No legal insertion point found.\n");
342 TBB = FBB = Tail =
nullptr;
344 if (Head->succ_size() != 2)
364 DEBUG(
dbgs() <<
"\nDiamond: BB#" << Head->getNumber()
367 <<
" -> BB#" << Tail->getNumber() <<
'\n');
370 if (!Tail->livein_empty()) {
375 DEBUG(
dbgs() <<
"\nTriangle: BB#" << Head->getNumber()
377 <<
" -> BB#" << Tail->getNumber() <<
'\n');
382 if (Tail->empty() || !Tail->front().isPHI()) {
390 DEBUG(
dbgs() <<
"Branch not analyzable.\n");
396 DEBUG(
dbgs() <<
"AnalyzeBranch didn't find conditional branch.\n");
402 FBB = TBB == Succ0 ? Succ1 : Succ0;
409 I !=
E && I->isPHI(); ++
I) {
411 PHIInfo &PI = PHIs.back();
413 for (
unsigned i = 1;
i != PI.PHI->getNumOperands();
i += 2) {
414 if (PI.PHI->getOperand(
i+1).getMBB() == TPred)
415 PI.TReg = PI.PHI->getOperand(
i).getReg();
416 if (PI.PHI->getOperand(
i+1).getMBB() == FPred)
417 PI.FReg = PI.PHI->getOperand(
i).getReg();
423 if (!
TII->canInsertSelect(*Head, Cond, PI.TReg, PI.FReg,
424 PI.CondCycles, PI.TCycles, PI.FCycles)) {
425 DEBUG(
dbgs() <<
"Can't convert: " << *PI.PHI);
432 ClobberedRegUnits.reset();
433 if (TBB != Tail && !canSpeculateInstrs(TBB))
435 if (FBB != Tail && !canSpeculateInstrs(FBB))
440 if (!findInsertionPoint())
453 void SSAIfConv::replacePHIInstrs() {
454 assert(Tail->pred_size() == 2 &&
"Cannot replace PHIs");
456 assert(FirstTerm != Head->end() &&
"No terminators");
457 DebugLoc HeadDL = FirstTerm->getDebugLoc();
460 for (
unsigned i = 0, e = PHIs.size();
i != e; ++
i) {
461 PHIInfo &PI = PHIs[
i];
462 DEBUG(
dbgs() <<
"If-converting " << *PI.PHI);
463 unsigned DstReg = PI.PHI->getOperand(0).getReg();
464 TII->insertSelect(*Head, FirstTerm, HeadDL, DstReg, Cond, PI.TReg, PI.FReg);
465 DEBUG(
dbgs() <<
" --> " << *std::prev(FirstTerm));
466 PI.PHI->eraseFromParent();
474 void SSAIfConv::rewritePHIOperands() {
476 assert(FirstTerm != Head->end() &&
"No terminators");
477 DebugLoc HeadDL = FirstTerm->getDebugLoc();
480 for (
unsigned i = 0, e = PHIs.size();
i != e; ++
i) {
481 PHIInfo &PI = PHIs[
i];
484 DEBUG(
dbgs() <<
"If-converting " << *PI.PHI);
485 if (PI.TReg == PI.FReg) {
490 unsigned PHIDst = PI.PHI->getOperand(0).getReg();
491 DstReg =
MRI->createVirtualRegister(
MRI->getRegClass(PHIDst));
492 TII->insertSelect(*Head, FirstTerm, HeadDL,
493 DstReg, Cond, PI.TReg, PI.FReg);
494 DEBUG(
dbgs() <<
" --> " << *std::prev(FirstTerm));
498 for (
unsigned i = PI.PHI->getNumOperands();
i != 1;
i -= 2) {
500 if (MBB == getTPred()) {
501 PI.PHI->getOperand(
i-1).setMBB(Head);
502 PI.PHI->getOperand(
i-2).setReg(DstReg);
503 }
else if (MBB == getFPred()) {
504 PI.PHI->RemoveOperand(
i-1);
505 PI.PHI->RemoveOperand(
i-2);
518 assert(Head && Tail && TBB && FBB &&
"Call canConvertIf first.");
528 Head->splice(InsertionPoint, TBB, TBB->begin(), TBB->getFirstTerminator());
530 Head->splice(InsertionPoint, FBB, FBB->begin(), FBB->getFirstTerminator());
533 bool ExtraPreds = Tail->pred_size() != 2;
535 rewritePHIOperands();
540 Head->removeSuccessor(TBB);
541 Head->removeSuccessor(FBB,
true);
543 TBB->removeSuccessor(Tail,
true);
545 FBB->removeSuccessor(Tail,
true);
549 DebugLoc HeadDL = Head->getFirstTerminator()->getDebugLoc();
556 TBB->eraseFromParent();
560 FBB->eraseFromParent();
563 assert(Head->succ_empty() &&
"Additional head successors?");
564 if (!ExtraPreds && Head->isLayoutSuccessor(Tail)) {
566 DEBUG(
dbgs() <<
"Joining tail BB#" << Tail->getNumber()
567 <<
" into head BB#" << Head->getNumber() <<
'\n');
568 Head->splice(Head->end(), Tail,
569 Tail->begin(), Tail->end());
570 Head->transferSuccessorsAndUpdatePHIs(Tail);
572 Tail->eraseFromParent();
575 DEBUG(
dbgs() <<
"Converting to unconditional branch.\n");
578 Head->addSuccessor(Tail);
605 StringRef getPassName()
const override {
return "Early If-Conversion"; }
611 void invalidateTraces();
612 bool shouldConvertIf();
620 "early-ifcvt",
"Early If Converter",
false,
false)
627 void EarlyIfConverter::getAnalysisUsage(
AnalysisUsage &AU)
const {
644 for (
unsigned i = 0, e = Removed.
size();
i != e; ++
i) {
646 assert(Node != HeadNode &&
"Cannot erase the head node");
649 DomTree->changeImmediateDominator(Node->
getChildren().back(), HeadNode);
651 DomTree->eraseNode(Removed[i]);
661 for (
unsigned i = 0, e = Removed.
size(); i != e; ++
i)
662 Loops->removeBlock(Removed[i]);
666 void EarlyIfConverter::invalidateTraces() {
667 Traces->verifyAnalysis();
668 Traces->invalidate(IfConv.Head);
669 Traces->invalidate(IfConv.Tail);
670 Traces->invalidate(IfConv.TBB);
671 Traces->invalidate(IfConv.FBB);
672 Traces->verifyAnalysis();
677 if (Delta < 0 && Cyc + Delta > Cyc)
685 bool EarlyIfConverter::shouldConvertIf() {
695 DEBUG(
dbgs() <<
"TBB: " << TBBTrace <<
"FBB: " << FBBTrace);
700 unsigned CritLimit = SchedModel.MispredictPenalty/2;
706 if (IfConv.TBB != IfConv.Tail)
709 DEBUG(
dbgs() <<
"Resource length " << ResLength
710 <<
", minimal critical path " << MinCrit <<
'\n');
711 if (ResLength > MinCrit + CritLimit) {
712 DEBUG(
dbgs() <<
"Not enough available ILP.\n");
720 unsigned BranchDepth =
722 DEBUG(
dbgs() <<
"Branch depth: " << BranchDepth <<
'\n');
727 for (
unsigned i = 0, e = IfConv.PHIs.size(); i != e; ++
i) {
728 SSAIfConv::PHIInfo &PI = IfConv.PHIs[
i];
731 DEBUG(
dbgs() <<
"Slack " << Slack <<
":\t" << *PI.PHI);
734 unsigned CondDepth =
adjCycles(BranchDepth, PI.CondCycles);
735 if (CondDepth > MaxDepth) {
736 unsigned Extra = CondDepth -
MaxDepth;
737 DEBUG(
dbgs() <<
"Condition adds " << Extra <<
" cycles.\n");
738 if (Extra > CritLimit) {
739 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
746 if (TDepth > MaxDepth) {
748 DEBUG(
dbgs() <<
"TBB data adds " << Extra <<
" cycles.\n");
749 if (Extra > CritLimit) {
750 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
757 if (FDepth > MaxDepth) {
759 DEBUG(
dbgs() <<
"FBB data adds " << Extra <<
" cycles.\n");
760 if (Extra > CritLimit) {
761 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
772 bool Changed =
false;
773 while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
777 IfConv.convertIf(RemovedBlocks);
779 updateDomTree(RemovedBlocks);
780 updateLoops(RemovedBlocks);
786 DEBUG(
dbgs() <<
"********** EARLY IF-CONVERSION **********\n"
787 <<
"********** Function: " << MF.
getName() <<
'\n');
800 DomTree = &getAnalysis<MachineDominatorTree>();
801 Loops = getAnalysisIfAvailable<MachineLoopInfo>();
802 Traces = &getAnalysis<MachineTraceMetrics>();
805 bool Changed =
false;
806 IfConv.runOnMachineFunction(MF);
813 if (tryConvertIf(DomNode->getBlock()))
unsigned succ_size() const
void push_back(const T &Elt)
static cl::opt< bool > Stress("stress-early-ifcvt", cl::Hidden, cl::desc("Turn all knobs to 11"))
const std::vector< DomTreeNodeBase< NodeT > * > & getChildren() const
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned Depth
Earliest issue cycle as determined by data dependencies and instruction latencies from the beginning ...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned getCriticalPath() const
Return the length of the (data dependency) critical path through the trace.
char & EarlyIfConverterID
EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions...
A trace ensemble is a collection of traces selected using the same strategy, for example 'minimum res...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
static unsigned InstrCount
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
InstrCycles getInstrCycles(const MachineInstr &MI) const
Return the depth and height of MI.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
#define INITIALIZE_PASS_DEPENDENCY(depName)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Reg
All possible values of the reg field in the ModR/M byte.
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
LLVM_NODISCARD bool empty() const
Select the trace through a block that has the fewest instructions.
bool livein_empty() const
Base class for the actual dominator tree node.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
size_t size() const
size - Get the array size.
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
initializer< Ty > init(const Ty &Val)
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned getResourceLength(ArrayRef< const MachineBasicBlock * > Extrablocks=None, ArrayRef< const MCSchedClassDesc * > ExtraInstrs=None, ArrayRef< const MCSchedClassDesc * > RemoveInstrs=None) const
Return the resource length of the trace.
virtual bool enableEarlyIfConversion() const
Enable the use of the early if conversion pass.
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
iterator_range< po_iterator< T > > post_order(const T &G)
succ_iterator succ_begin()
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A trace represents a plausible sequence of executed basic blocks that passes through the current basi...
Printable PrintRegUnit(unsigned Unit, const TargetRegisterInfo *TRI)
Create Printable object to print register units on a raw_ostream.
static unsigned adjCycles(unsigned Cyc, int Delta)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
unsigned getPHIDepth(const MachineInstr &PHI) const
Return the Depth of a PHI instruction in a trace center block successor.
LLVM_NODISCARD T pop_back_val()
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static cl::opt< unsigned > BlockInstrLimit("early-ifcvt-limit", cl::init(30), cl::Hidden, cl::desc("Maximum number of instructions per speculated block."))
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
INITIALIZE_PASS_BEGIN(EarlyIfConverter,"early-ifcvt","Early If Converter", false, false) INITIALIZE_PASS_END(EarlyIfConverter
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
Remove the branching code at the end of the specific MBB.
Maximum length of the test input If
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
unsigned getInstrSlack(const MachineInstr &MI) const
Return the slack of MI.
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Machine model for scheduling, bundling, and heuristics.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const
size_t getNumChildren() const