42 #define DEBUG_TYPE "early-ifcvt"
48 cl::desc(
"Maximum number of instructions per speculated block."));
54 STATISTIC(NumDiamondsSeen,
"Number of diamonds");
55 STATISTIC(NumDiamondsConv,
"Number of diamonds converted");
56 STATISTIC(NumTrianglesSeen,
"Number of triangles");
57 STATISTIC(NumTrianglesConv,
"Number of triangles converted");
101 bool isTriangle()
const {
return TBB == Tail || FBB == Tail; }
114 int CondCycles, TCycles, FCycles;
117 :
PHI(phi), TReg(0), FReg(0), CondCycles(0), TCycles(0), FCycles(0) {}
145 bool findInsertionPoint();
148 void replacePHIInstrs();
151 void rewritePHIOperands();
159 LiveRegUnits.clear();
160 LiveRegUnits.setUniverse(TRI->getNumRegUnits());
161 ClobberedRegUnits.clear();
162 ClobberedRegUnits.resize(TRI->getNumRegUnits());
192 unsigned InstrCount = 0;
198 if (
I->isDebugValue())
217 DEBUG(
dbgs() <<
"Won't speculate load: " << *
I);
222 bool DontMoveAcrossStore =
true;
223 if (!
I->isSafeToMove(
nullptr, DontMoveAcrossStore)) {
230 if (MO.isRegMask()) {
231 DEBUG(
dbgs() <<
"Won't speculate regmask: " << *
I);
236 unsigned Reg = MO.getReg();
241 ClobberedRegUnits.set(*Units);
246 if (!DefMI || DefMI->
getParent() != Head)
248 if (InsertAfter.insert(DefMI).second)
251 DEBUG(
dbgs() <<
"Can't insert instructions below terminator.\n");
270 bool SSAIfConv::findInsertionPoint() {
273 LiveRegUnits.clear();
281 if (InsertAfter.count(I)) {
282 DEBUG(
dbgs() <<
"Can't insert code after " << *I);
291 unsigned Reg = MO.getReg();
297 LiveRegUnits.erase(*Units);
303 while (!Reads.
empty())
306 if (ClobberedRegUnits.test(*Units))
307 LiveRegUnits.insert(*Units);
310 if (I != FirstTerm && I->isTerminator())
315 if (!LiveRegUnits.empty()) {
317 dbgs() <<
"Would clobber";
319 i = LiveRegUnits.
begin(), e = LiveRegUnits.end(); i != e; ++i)
321 dbgs() <<
" live before " << *
I;
328 DEBUG(
dbgs() <<
"Can insert before " << *I);
331 DEBUG(
dbgs() <<
"No legal insertion point found.\n");
342 TBB = FBB = Tail =
nullptr;
344 if (Head->succ_size() != 2)
364 DEBUG(
dbgs() <<
"\nDiamond: BB#" << Head->getNumber()
367 <<
" -> BB#" << Tail->getNumber() <<
'\n');
370 if (!Tail->livein_empty()) {
375 DEBUG(
dbgs() <<
"\nTriangle: BB#" << Head->getNumber()
377 <<
" -> BB#" << Tail->getNumber() <<
'\n');
382 if (Tail->empty() || !Tail->front().isPHI()) {
390 DEBUG(
dbgs() <<
"Branch not analyzable.\n");
396 DEBUG(
dbgs() <<
"AnalyzeBranch didn't find conditional branch.\n");
402 FBB = TBB == Succ0 ? Succ1 : Succ0;
409 I != E && I->isPHI(); ++
I) {
411 PHIInfo &PI = PHIs.back();
413 for (
unsigned i = 1; i != PI.PHI->getNumOperands(); i += 2) {
414 if (PI.PHI->getOperand(i+1).getMBB() == TPred)
415 PI.TReg = PI.PHI->getOperand(i).getReg();
416 if (PI.PHI->getOperand(i+1).getMBB() == FPred)
417 PI.FReg = PI.PHI->getOperand(i).getReg();
423 if (!
TII->canInsertSelect(*Head, Cond, PI.TReg, PI.FReg,
424 PI.CondCycles, PI.TCycles, PI.FCycles)) {
425 DEBUG(
dbgs() <<
"Can't convert: " << *PI.PHI);
432 ClobberedRegUnits.reset();
433 if (TBB != Tail && !canSpeculateInstrs(TBB))
435 if (FBB != Tail && !canSpeculateInstrs(FBB))
440 if (!findInsertionPoint())
453 void SSAIfConv::replacePHIInstrs() {
454 assert(Tail->pred_size() == 2 &&
"Cannot replace PHIs");
456 assert(FirstTerm != Head->end() &&
"No terminators");
457 DebugLoc HeadDL = FirstTerm->getDebugLoc();
460 for (
unsigned i = 0, e = PHIs.size(); i != e; ++i) {
461 PHIInfo &PI = PHIs[i];
462 DEBUG(
dbgs() <<
"If-converting " << *PI.PHI);
463 unsigned DstReg = PI.PHI->getOperand(0).getReg();
464 TII->insertSelect(*Head, FirstTerm, HeadDL, DstReg, Cond, PI.TReg, PI.FReg);
465 DEBUG(
dbgs() <<
" --> " << *std::prev(FirstTerm));
466 PI.PHI->eraseFromParent();
474 void SSAIfConv::rewritePHIOperands() {
476 assert(FirstTerm != Head->end() &&
"No terminators");
477 DebugLoc HeadDL = FirstTerm->getDebugLoc();
480 for (
unsigned i = 0, e = PHIs.size(); i != e; ++i) {
481 PHIInfo &PI = PHIs[i];
484 DEBUG(
dbgs() <<
"If-converting " << *PI.PHI);
485 if (PI.TReg == PI.FReg) {
490 unsigned PHIDst = PI.PHI->getOperand(0).getReg();
491 DstReg = MRI->createVirtualRegister(MRI->getRegClass(PHIDst));
492 TII->insertSelect(*Head, FirstTerm, HeadDL,
493 DstReg, Cond, PI.TReg, PI.FReg);
494 DEBUG(
dbgs() <<
" --> " << *std::prev(FirstTerm));
498 for (
unsigned i = PI.PHI->getNumOperands(); i != 1; i -= 2) {
500 if (MBB == getTPred()) {
501 PI.PHI->getOperand(i-1).setMBB(Head);
502 PI.PHI->getOperand(i-2).setReg(DstReg);
503 }
else if (MBB == getFPred()) {
504 PI.PHI->RemoveOperand(i-1);
505 PI.PHI->RemoveOperand(i-2);
518 assert(Head && Tail && TBB && FBB &&
"Call canConvertIf first.");
528 Head->splice(InsertionPoint, TBB, TBB->begin(), TBB->getFirstTerminator());
530 Head->splice(InsertionPoint, FBB, FBB->begin(), FBB->getFirstTerminator());
533 bool ExtraPreds = Tail->pred_size() != 2;
535 rewritePHIOperands();
540 Head->removeSuccessor(TBB);
541 Head->removeSuccessor(FBB);
543 TBB->removeSuccessor(Tail);
545 FBB->removeSuccessor(Tail);
549 DebugLoc HeadDL = Head->getFirstTerminator()->getDebugLoc();
556 TBB->eraseFromParent();
560 FBB->eraseFromParent();
563 assert(Head->succ_empty() &&
"Additional head successors?");
564 if (!ExtraPreds && Head->isLayoutSuccessor(Tail)) {
566 DEBUG(
dbgs() <<
"Joining tail BB#" << Tail->getNumber()
567 <<
" into head BB#" << Head->getNumber() <<
'\n');
568 Head->splice(Head->end(), Tail,
569 Tail->begin(), Tail->end());
570 Head->transferSuccessorsAndUpdatePHIs(Tail);
572 Tail->eraseFromParent();
575 DEBUG(
dbgs() <<
"Converting to unconditional branch.\n");
578 Head->addSuccessor(Tail);
605 const char *getPassName()
const override {
return "Early If-Conversion"; }
611 void invalidateTraces();
612 bool shouldConvertIf();
620 "early-ifcvt",
"Early If Converter",
false,
false)
627 void EarlyIfConverter::getAnalysisUsage(
AnalysisUsage &AU)
const {
644 for (
unsigned i = 0, e = Removed.
size(); i != e; ++i) {
646 assert(Node != HeadNode &&
"Cannot erase the head node");
648 assert(Node->
getBlock() == IfConv.Tail &&
"Unexpected children");
649 DomTree->changeImmediateDominator(Node->
getChildren().back(), HeadNode);
651 DomTree->eraseNode(Removed[i]);
661 for (
unsigned i = 0, e = Removed.
size(); i != e; ++i)
662 Loops->removeBlock(Removed[i]);
666 void EarlyIfConverter::invalidateTraces() {
667 Traces->verifyAnalysis();
668 Traces->invalidate(IfConv.Head);
669 Traces->invalidate(IfConv.Tail);
670 Traces->invalidate(IfConv.TBB);
671 Traces->invalidate(IfConv.FBB);
672 Traces->verifyAnalysis();
677 if (Delta < 0 && Cyc + Delta > Cyc)
685 bool EarlyIfConverter::shouldConvertIf() {
695 DEBUG(
dbgs() <<
"TBB: " << TBBTrace <<
"FBB: " << FBBTrace);
700 unsigned CritLimit = SchedModel.MispredictPenalty/2;
706 if (IfConv.TBB != IfConv.Tail)
709 DEBUG(
dbgs() <<
"Resource length " << ResLength
710 <<
", minimal critical path " << MinCrit <<
'\n');
711 if (ResLength > MinCrit + CritLimit) {
712 DEBUG(
dbgs() <<
"Not enough available ILP.\n");
720 unsigned BranchDepth =
721 HeadTrace.
getInstrCycles(IfConv.Head->getFirstTerminator()).Depth;
722 DEBUG(
dbgs() <<
"Branch depth: " << BranchDepth <<
'\n');
727 for (
unsigned i = 0, e = IfConv.PHIs.size(); i != e; ++i) {
728 SSAIfConv::PHIInfo &PI = IfConv.PHIs[i];
731 DEBUG(
dbgs() <<
"Slack " << Slack <<
":\t" << *PI.PHI);
734 unsigned CondDepth =
adjCycles(BranchDepth, PI.CondCycles);
735 if (CondDepth > MaxDepth) {
736 unsigned Extra = CondDepth -
MaxDepth;
737 DEBUG(
dbgs() <<
"Condition adds " << Extra <<
" cycles.\n");
738 if (Extra > CritLimit) {
739 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
746 if (TDepth > MaxDepth) {
748 DEBUG(
dbgs() <<
"TBB data adds " << Extra <<
" cycles.\n");
749 if (Extra > CritLimit) {
750 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
757 if (FDepth > MaxDepth) {
759 DEBUG(
dbgs() <<
"FBB data adds " << Extra <<
" cycles.\n");
760 if (Extra > CritLimit) {
761 DEBUG(
dbgs() <<
"Exceeds limit of " << CritLimit <<
'\n');
772 bool Changed =
false;
773 while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
777 IfConv.convertIf(RemovedBlocks);
779 updateDomTree(RemovedBlocks);
780 updateLoops(RemovedBlocks);
786 DEBUG(
dbgs() <<
"********** EARLY IF-CONVERSION **********\n"
787 <<
"********** Function: " << MF.
getName() <<
'\n');
790 if (!STI.enableEarlyIfConversion())
793 TII = STI.getInstrInfo();
795 SchedModel = STI.getSchedModel();
797 DomTree = &getAnalysis<MachineDominatorTree>();
798 Loops = getAnalysisIfAvailable<MachineLoopInfo>();
799 Traces = &getAnalysis<MachineTraceMetrics>();
802 bool Changed =
false;
803 IfConv.runOnMachineFunction(MF);
810 if (tryConvertIf(DomNode->getBlock()))
unsigned succ_size() const
void push_back(const T &Elt)
bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
static cl::opt< bool > Stress("stress-early-ifcvt", cl::Hidden, cl::desc("Turn all knobs to 11"))
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
unsigned Depth
Earliest issue cycle as determined by data dependencies and instruction latencies from the beginning ...
iterator getFirstTerminator()
getFirstTerminator - returns an iterator to the first terminator instruction of this basic block...
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
PrintRegUnit - Helper class for printing register units on a raw_ostream.
unsigned getCriticalPath() const
Return the length of the (data dependency) critical path through the trace.
char & EarlyIfConverterID
EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions...
A trace ensemble is a collection of traces selected using the same strategy, for example 'minimum res...
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
#define INITIALIZE_PASS_DEPENDENCY(depName)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Reg
All possible values of the reg field in the ModR/M byte.
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
Number of individual test Apply this number of consecutive mutations to each input If
Select the trace through a block that has the fewest instructions.
const HexagonRegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
bool livein_empty() const
Base class for the actual dominator tree node.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
size_t size() const
size - Get the array size.
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned RemoveBranch(MachineBasicBlock &MBB) const override
unsigned getResourceLength(ArrayRef< const MachineBasicBlock * > Extrablocks=None, ArrayRef< const MCSchedClassDesc * > ExtraInstrs=None, ArrayRef< const MCSchedClassDesc * > RemoveInstrs=None) const
Return the resource length of the trace.
Represent the analysis usage information of a pass.
iterator_range< po_iterator< T > > post_order(const T &G)
const std::vector< DomTreeNodeBase< NodeT > * > & getChildren() const
succ_iterator succ_begin()
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A trace represents a plausible sequence of executed basic blocks that passes through the current basi...
const_iterator begin() const
static unsigned adjCycles(unsigned Cyc, int Delta)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static cl::opt< unsigned > BlockInstrLimit("early-ifcvt-limit", cl::init(30), cl::Hidden, cl::desc("Maximum number of instructions per speculated block."))
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
INITIALIZE_PASS_BEGIN(EarlyIfConverter,"early-ifcvt","Early If Converter", false, false) INITIALIZE_PASS_END(EarlyIfConverter
size_t getNumChildren() const
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
InstrCycles getInstrCycles(const MachineInstr *MI) const
Return the depth and height of MI.
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, DebugLoc DL) const override
unsigned getPHIDepth(const MachineInstr *PHI) const
Return the Depth of a PHI instruction in a trace center block successor.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Machine model for scheduling, bundling, and heuristics.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const
unsigned getInstrSlack(const MachineInstr *MI) const
Return the slack of MI.