LLVM  4.0.0
EarlyIfConversion.cpp
Go to the documentation of this file.
1 //===-- EarlyIfConversion.cpp - If-conversion on SSA form machine code ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Early if-conversion is for out-of-order CPUs that don't have a lot of
11 // predicable instructions. The goal is to eliminate conditional branches that
12 // may mispredict.
13 //
14 // Instructions from both sides of the branch are executed specutatively, and a
15 // cmov instruction selects the result.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "llvm/ADT/BitVector.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/SmallPtrSet.h"
23 #include "llvm/ADT/SparseSet.h"
24 #include "llvm/ADT/Statistic.h"
32 #include "llvm/CodeGen/Passes.h"
34 #include "llvm/Support/Debug.h"
39 
40 using namespace llvm;
41 
42 #define DEBUG_TYPE "early-ifcvt"
43 
44 // Absolute maximum number of instructions allowed per speculated block.
45 // This bypasses all other heuristics, so it should be set fairly high.
46 static cl::opt<unsigned>
47 BlockInstrLimit("early-ifcvt-limit", cl::init(30), cl::Hidden,
48  cl::desc("Maximum number of instructions per speculated block."));
49 
50 // Stress testing mode - disable heuristics.
51 static cl::opt<bool> Stress("stress-early-ifcvt", cl::Hidden,
52  cl::desc("Turn all knobs to 11"));
53 
54 STATISTIC(NumDiamondsSeen, "Number of diamonds");
55 STATISTIC(NumDiamondsConv, "Number of diamonds converted");
56 STATISTIC(NumTrianglesSeen, "Number of triangles");
57 STATISTIC(NumTrianglesConv, "Number of triangles converted");
58 
59 //===----------------------------------------------------------------------===//
60 // SSAIfConv
61 //===----------------------------------------------------------------------===//
62 //
63 // The SSAIfConv class performs if-conversion on SSA form machine code after
64 // determining if it is possible. The class contains no heuristics; external
65 // code should be used to determine when if-conversion is a good idea.
66 //
67 // SSAIfConv can convert both triangles and diamonds:
68 //
69 // Triangle: Head Diamond: Head
70 // | \ / \_
71 // | \ / |
72 // | [TF]BB FBB TBB
73 // | / \ /
74 // | / \ /
75 // Tail Tail
76 //
77 // Instructions in the conditional blocks TBB and/or FBB are spliced into the
78 // Head block, and phis in the Tail block are converted to select instructions.
79 //
80 namespace {
81 class SSAIfConv {
82  const TargetInstrInfo *TII;
83  const TargetRegisterInfo *TRI;
85 
86 public:
87  /// The block containing the conditional branch.
88  MachineBasicBlock *Head;
89 
90  /// The block containing phis after the if-then-else.
91  MachineBasicBlock *Tail;
92 
93  /// The 'true' conditional block as determined by AnalyzeBranch.
94  MachineBasicBlock *TBB;
95 
96  /// The 'false' conditional block as determined by AnalyzeBranch.
97  MachineBasicBlock *FBB;
98 
99  /// isTriangle - When there is no 'else' block, either TBB or FBB will be
100  /// equal to Tail.
101  bool isTriangle() const { return TBB == Tail || FBB == Tail; }
102 
103  /// Returns the Tail predecessor for the True side.
104  MachineBasicBlock *getTPred() const { return TBB == Tail ? Head : TBB; }
105 
106  /// Returns the Tail predecessor for the False side.
107  MachineBasicBlock *getFPred() const { return FBB == Tail ? Head : FBB; }
108 
109  /// Information about each phi in the Tail block.
110  struct PHIInfo {
111  MachineInstr *PHI;
112  unsigned TReg, FReg;
113  // Latencies from Cond+Branch, TReg, and FReg to DstReg.
114  int CondCycles, TCycles, FCycles;
115 
116  PHIInfo(MachineInstr *phi)
117  : PHI(phi), TReg(0), FReg(0), CondCycles(0), TCycles(0), FCycles(0) {}
118  };
119 
121 
122 private:
123  /// The branch condition determined by AnalyzeBranch.
125 
126  /// Instructions in Head that define values used by the conditional blocks.
127  /// The hoisted instructions must be inserted after these instructions.
128  SmallPtrSet<MachineInstr*, 8> InsertAfter;
129 
130  /// Register units clobbered by the conditional blocks.
131  BitVector ClobberedRegUnits;
132 
133  // Scratch pad for findInsertionPoint.
134  SparseSet<unsigned> LiveRegUnits;
135 
136  /// Insertion point in Head for speculatively executed instructions form TBB
137  /// and FBB.
138  MachineBasicBlock::iterator InsertionPoint;
139 
140  /// Return true if all non-terminator instructions in MBB can be safely
141  /// speculated.
142  bool canSpeculateInstrs(MachineBasicBlock *MBB);
143 
144  /// Find a valid insertion point in Head.
145  bool findInsertionPoint();
146 
147  /// Replace PHI instructions in Tail with selects.
148  void replacePHIInstrs();
149 
150  /// Insert selects and rewrite PHI operands to use them.
151  void rewritePHIOperands();
152 
153 public:
154  /// runOnMachineFunction - Initialize per-function data structures.
155  void runOnMachineFunction(MachineFunction &MF) {
156  TII = MF.getSubtarget().getInstrInfo();
157  TRI = MF.getSubtarget().getRegisterInfo();
158  MRI = &MF.getRegInfo();
159  LiveRegUnits.clear();
160  LiveRegUnits.setUniverse(TRI->getNumRegUnits());
161  ClobberedRegUnits.clear();
162  ClobberedRegUnits.resize(TRI->getNumRegUnits());
163  }
164 
165  /// canConvertIf - If the sub-CFG headed by MBB can be if-converted,
166  /// initialize the internal state, and return true.
167  bool canConvertIf(MachineBasicBlock *MBB);
168 
169  /// convertIf - If-convert the last block passed to canConvertIf(), assuming
170  /// it is possible. Add any erased blocks to RemovedBlocks.
171  void convertIf(SmallVectorImpl<MachineBasicBlock*> &RemovedBlocks);
172 };
173 } // end anonymous namespace
174 
175 
176 /// canSpeculateInstrs - Returns true if all the instructions in MBB can safely
177 /// be speculated. The terminators are not considered.
178 ///
179 /// If instructions use any values that are defined in the head basic block,
180 /// the defining instructions are added to InsertAfter.
181 ///
182 /// Any clobbered regunits are added to ClobberedRegUnits.
183 ///
184 bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
185  // Reject any live-in physregs. It's probably CPSR/EFLAGS, and very hard to
186  // get right.
187  if (!MBB->livein_empty()) {
188  DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has live-ins.\n");
189  return false;
190  }
191 
192  unsigned InstrCount = 0;
193 
194  // Check all instructions, except the terminators. It is assumed that
195  // terminators never have side effects or define any used register values.
196  for (MachineBasicBlock::iterator I = MBB->begin(),
197  E = MBB->getFirstTerminator(); I != E; ++I) {
198  if (I->isDebugValue())
199  continue;
200 
201  if (++InstrCount > BlockInstrLimit && !Stress) {
202  DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has more than "
203  << BlockInstrLimit << " instructions.\n");
204  return false;
205  }
206 
207  // There shouldn't normally be any phis in a single-predecessor block.
208  if (I->isPHI()) {
209  DEBUG(dbgs() << "Can't hoist: " << *I);
210  return false;
211  }
212 
213  // Don't speculate loads. Note that it may be possible and desirable to
214  // speculate GOT or constant pool loads that are guaranteed not to trap,
215  // but we don't support that for now.
216  if (I->mayLoad()) {
217  DEBUG(dbgs() << "Won't speculate load: " << *I);
218  return false;
219  }
220 
221  // We never speculate stores, so an AA pointer isn't necessary.
222  bool DontMoveAcrossStore = true;
223  if (!I->isSafeToMove(nullptr, DontMoveAcrossStore)) {
224  DEBUG(dbgs() << "Can't speculate: " << *I);
225  return false;
226  }
227 
228  // Check for any dependencies on Head instructions.
229  for (const MachineOperand &MO : I->operands()) {
230  if (MO.isRegMask()) {
231  DEBUG(dbgs() << "Won't speculate regmask: " << *I);
232  return false;
233  }
234  if (!MO.isReg())
235  continue;
236  unsigned Reg = MO.getReg();
237 
238  // Remember clobbered regunits.
239  if (MO.isDef() && TargetRegisterInfo::isPhysicalRegister(Reg))
240  for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
241  ClobberedRegUnits.set(*Units);
242 
243  if (!MO.readsReg() || !TargetRegisterInfo::isVirtualRegister(Reg))
244  continue;
245  MachineInstr *DefMI = MRI->getVRegDef(Reg);
246  if (!DefMI || DefMI->getParent() != Head)
247  continue;
248  if (InsertAfter.insert(DefMI).second)
249  DEBUG(dbgs() << "BB#" << MBB->getNumber() << " depends on " << *DefMI);
250  if (DefMI->isTerminator()) {
251  DEBUG(dbgs() << "Can't insert instructions below terminator.\n");
252  return false;
253  }
254  }
255  }
256  return true;
257 }
258 
259 
260 /// Find an insertion point in Head for the speculated instructions. The
261 /// insertion point must be:
262 ///
263 /// 1. Before any terminators.
264 /// 2. After any instructions in InsertAfter.
265 /// 3. Not have any clobbered regunits live.
266 ///
267 /// This function sets InsertionPoint and returns true when successful, it
268 /// returns false if no valid insertion point could be found.
269 ///
270 bool SSAIfConv::findInsertionPoint() {
271  // Keep track of live regunits before the current position.
272  // Only track RegUnits that are also in ClobberedRegUnits.
273  LiveRegUnits.clear();
275  MachineBasicBlock::iterator FirstTerm = Head->getFirstTerminator();
276  MachineBasicBlock::iterator I = Head->end();
277  MachineBasicBlock::iterator B = Head->begin();
278  while (I != B) {
279  --I;
280  // Some of the conditional code depends in I.
281  if (InsertAfter.count(&*I)) {
282  DEBUG(dbgs() << "Can't insert code after " << *I);
283  return false;
284  }
285 
286  // Update live regunits.
287  for (const MachineOperand &MO : I->operands()) {
288  // We're ignoring regmask operands. That is conservatively correct.
289  if (!MO.isReg())
290  continue;
291  unsigned Reg = MO.getReg();
293  continue;
294  // I clobbers Reg, so it isn't live before I.
295  if (MO.isDef())
296  for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
297  LiveRegUnits.erase(*Units);
298  // Unless I reads Reg.
299  if (MO.readsReg())
300  Reads.push_back(Reg);
301  }
302  // Anything read by I is live before I.
303  while (!Reads.empty())
304  for (MCRegUnitIterator Units(Reads.pop_back_val(), TRI); Units.isValid();
305  ++Units)
306  if (ClobberedRegUnits.test(*Units))
307  LiveRegUnits.insert(*Units);
308 
309  // We can't insert before a terminator.
310  if (I != FirstTerm && I->isTerminator())
311  continue;
312 
313  // Some of the clobbered registers are live before I, not a valid insertion
314  // point.
315  if (!LiveRegUnits.empty()) {
316  DEBUG({
317  dbgs() << "Would clobber";
319  i = LiveRegUnits.begin(), e = LiveRegUnits.end(); i != e; ++i)
320  dbgs() << ' ' << PrintRegUnit(*i, TRI);
321  dbgs() << " live before " << *I;
322  });
323  continue;
324  }
325 
326  // This is a valid insertion point.
327  InsertionPoint = I;
328  DEBUG(dbgs() << "Can insert before " << *I);
329  return true;
330  }
331  DEBUG(dbgs() << "No legal insertion point found.\n");
332  return false;
333 }
334 
335 
336 
337 /// canConvertIf - analyze the sub-cfg rooted in MBB, and return true if it is
338 /// a potential candidate for if-conversion. Fill out the internal state.
339 ///
340 bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB) {
341  Head = MBB;
342  TBB = FBB = Tail = nullptr;
343 
344  if (Head->succ_size() != 2)
345  return false;
346  MachineBasicBlock *Succ0 = Head->succ_begin()[0];
347  MachineBasicBlock *Succ1 = Head->succ_begin()[1];
348 
349  // Canonicalize so Succ0 has MBB as its single predecessor.
350  if (Succ0->pred_size() != 1)
351  std::swap(Succ0, Succ1);
352 
353  if (Succ0->pred_size() != 1 || Succ0->succ_size() != 1)
354  return false;
355 
356  Tail = Succ0->succ_begin()[0];
357 
358  // This is not a triangle.
359  if (Tail != Succ1) {
360  // Check for a diamond. We won't deal with any critical edges.
361  if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
362  Succ1->succ_begin()[0] != Tail)
363  return false;
364  DEBUG(dbgs() << "\nDiamond: BB#" << Head->getNumber()
365  << " -> BB#" << Succ0->getNumber()
366  << "/BB#" << Succ1->getNumber()
367  << " -> BB#" << Tail->getNumber() << '\n');
368 
369  // Live-in physregs are tricky to get right when speculating code.
370  if (!Tail->livein_empty()) {
371  DEBUG(dbgs() << "Tail has live-ins.\n");
372  return false;
373  }
374  } else {
375  DEBUG(dbgs() << "\nTriangle: BB#" << Head->getNumber()
376  << " -> BB#" << Succ0->getNumber()
377  << " -> BB#" << Tail->getNumber() << '\n');
378  }
379 
380  // This is a triangle or a diamond.
381  // If Tail doesn't have any phis, there must be side effects.
382  if (Tail->empty() || !Tail->front().isPHI()) {
383  DEBUG(dbgs() << "No phis in tail.\n");
384  return false;
385  }
386 
387  // The branch we're looking to eliminate must be analyzable.
388  Cond.clear();
389  if (TII->analyzeBranch(*Head, TBB, FBB, Cond)) {
390  DEBUG(dbgs() << "Branch not analyzable.\n");
391  return false;
392  }
393 
394  // This is weird, probably some sort of degenerate CFG.
395  if (!TBB) {
396  DEBUG(dbgs() << "AnalyzeBranch didn't find conditional branch.\n");
397  return false;
398  }
399 
400  // AnalyzeBranch doesn't set FBB on a fall-through branch.
401  // Make sure it is always set.
402  FBB = TBB == Succ0 ? Succ1 : Succ0;
403 
404  // Any phis in the tail block must be convertible to selects.
405  PHIs.clear();
406  MachineBasicBlock *TPred = getTPred();
407  MachineBasicBlock *FPred = getFPred();
408  for (MachineBasicBlock::iterator I = Tail->begin(), E = Tail->end();
409  I != E && I->isPHI(); ++I) {
410  PHIs.push_back(&*I);
411  PHIInfo &PI = PHIs.back();
412  // Find PHI operands corresponding to TPred and FPred.
413  for (unsigned i = 1; i != PI.PHI->getNumOperands(); i += 2) {
414  if (PI.PHI->getOperand(i+1).getMBB() == TPred)
415  PI.TReg = PI.PHI->getOperand(i).getReg();
416  if (PI.PHI->getOperand(i+1).getMBB() == FPred)
417  PI.FReg = PI.PHI->getOperand(i).getReg();
418  }
419  assert(TargetRegisterInfo::isVirtualRegister(PI.TReg) && "Bad PHI");
420  assert(TargetRegisterInfo::isVirtualRegister(PI.FReg) && "Bad PHI");
421 
422  // Get target information.
423  if (!TII->canInsertSelect(*Head, Cond, PI.TReg, PI.FReg,
424  PI.CondCycles, PI.TCycles, PI.FCycles)) {
425  DEBUG(dbgs() << "Can't convert: " << *PI.PHI);
426  return false;
427  }
428  }
429 
430  // Check that the conditional instructions can be speculated.
431  InsertAfter.clear();
432  ClobberedRegUnits.reset();
433  if (TBB != Tail && !canSpeculateInstrs(TBB))
434  return false;
435  if (FBB != Tail && !canSpeculateInstrs(FBB))
436  return false;
437 
438  // Try to find a valid insertion point for the speculated instructions in the
439  // head basic block.
440  if (!findInsertionPoint())
441  return false;
442 
443  if (isTriangle())
444  ++NumTrianglesSeen;
445  else
446  ++NumDiamondsSeen;
447  return true;
448 }
449 
450 /// replacePHIInstrs - Completely replace PHI instructions with selects.
451 /// This is possible when the only Tail predecessors are the if-converted
452 /// blocks.
453 void SSAIfConv::replacePHIInstrs() {
454  assert(Tail->pred_size() == 2 && "Cannot replace PHIs");
455  MachineBasicBlock::iterator FirstTerm = Head->getFirstTerminator();
456  assert(FirstTerm != Head->end() && "No terminators");
457  DebugLoc HeadDL = FirstTerm->getDebugLoc();
458 
459  // Convert all PHIs to select instructions inserted before FirstTerm.
460  for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
461  PHIInfo &PI = PHIs[i];
462  DEBUG(dbgs() << "If-converting " << *PI.PHI);
463  unsigned DstReg = PI.PHI->getOperand(0).getReg();
464  TII->insertSelect(*Head, FirstTerm, HeadDL, DstReg, Cond, PI.TReg, PI.FReg);
465  DEBUG(dbgs() << " --> " << *std::prev(FirstTerm));
466  PI.PHI->eraseFromParent();
467  PI.PHI = nullptr;
468  }
469 }
470 
471 /// rewritePHIOperands - When there are additional Tail predecessors, insert
472 /// select instructions in Head and rewrite PHI operands to use the selects.
473 /// Keep the PHI instructions in Tail to handle the other predecessors.
474 void SSAIfConv::rewritePHIOperands() {
475  MachineBasicBlock::iterator FirstTerm = Head->getFirstTerminator();
476  assert(FirstTerm != Head->end() && "No terminators");
477  DebugLoc HeadDL = FirstTerm->getDebugLoc();
478 
479  // Convert all PHIs to select instructions inserted before FirstTerm.
480  for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
481  PHIInfo &PI = PHIs[i];
482  unsigned DstReg = 0;
483 
484  DEBUG(dbgs() << "If-converting " << *PI.PHI);
485  if (PI.TReg == PI.FReg) {
486  // We do not need the select instruction if both incoming values are
487  // equal.
488  DstReg = PI.TReg;
489  } else {
490  unsigned PHIDst = PI.PHI->getOperand(0).getReg();
491  DstReg = MRI->createVirtualRegister(MRI->getRegClass(PHIDst));
492  TII->insertSelect(*Head, FirstTerm, HeadDL,
493  DstReg, Cond, PI.TReg, PI.FReg);
494  DEBUG(dbgs() << " --> " << *std::prev(FirstTerm));
495  }
496 
497  // Rewrite PHI operands TPred -> (DstReg, Head), remove FPred.
498  for (unsigned i = PI.PHI->getNumOperands(); i != 1; i -= 2) {
499  MachineBasicBlock *MBB = PI.PHI->getOperand(i-1).getMBB();
500  if (MBB == getTPred()) {
501  PI.PHI->getOperand(i-1).setMBB(Head);
502  PI.PHI->getOperand(i-2).setReg(DstReg);
503  } else if (MBB == getFPred()) {
504  PI.PHI->RemoveOperand(i-1);
505  PI.PHI->RemoveOperand(i-2);
506  }
507  }
508  DEBUG(dbgs() << " --> " << *PI.PHI);
509  }
510 }
511 
512 /// convertIf - Execute the if conversion after canConvertIf has determined the
513 /// feasibility.
514 ///
515 /// Any basic blocks erased will be added to RemovedBlocks.
516 ///
517 void SSAIfConv::convertIf(SmallVectorImpl<MachineBasicBlock*> &RemovedBlocks) {
518  assert(Head && Tail && TBB && FBB && "Call canConvertIf first.");
519 
520  // Update statistics.
521  if (isTriangle())
522  ++NumTrianglesConv;
523  else
524  ++NumDiamondsConv;
525 
526  // Move all instructions into Head, except for the terminators.
527  if (TBB != Tail)
528  Head->splice(InsertionPoint, TBB, TBB->begin(), TBB->getFirstTerminator());
529  if (FBB != Tail)
530  Head->splice(InsertionPoint, FBB, FBB->begin(), FBB->getFirstTerminator());
531 
532  // Are there extra Tail predecessors?
533  bool ExtraPreds = Tail->pred_size() != 2;
534  if (ExtraPreds)
535  rewritePHIOperands();
536  else
537  replacePHIInstrs();
538 
539  // Fix up the CFG, temporarily leave Head without any successors.
540  Head->removeSuccessor(TBB);
541  Head->removeSuccessor(FBB, true);
542  if (TBB != Tail)
543  TBB->removeSuccessor(Tail, true);
544  if (FBB != Tail)
545  FBB->removeSuccessor(Tail, true);
546 
547  // Fix up Head's terminators.
548  // It should become a single branch or a fallthrough.
549  DebugLoc HeadDL = Head->getFirstTerminator()->getDebugLoc();
550  TII->removeBranch(*Head);
551 
552  // Erase the now empty conditional blocks. It is likely that Head can fall
553  // through to Tail, and we can join the two blocks.
554  if (TBB != Tail) {
555  RemovedBlocks.push_back(TBB);
556  TBB->eraseFromParent();
557  }
558  if (FBB != Tail) {
559  RemovedBlocks.push_back(FBB);
560  FBB->eraseFromParent();
561  }
562 
563  assert(Head->succ_empty() && "Additional head successors?");
564  if (!ExtraPreds && Head->isLayoutSuccessor(Tail)) {
565  // Splice Tail onto the end of Head.
566  DEBUG(dbgs() << "Joining tail BB#" << Tail->getNumber()
567  << " into head BB#" << Head->getNumber() << '\n');
568  Head->splice(Head->end(), Tail,
569  Tail->begin(), Tail->end());
570  Head->transferSuccessorsAndUpdatePHIs(Tail);
571  RemovedBlocks.push_back(Tail);
572  Tail->eraseFromParent();
573  } else {
574  // We need a branch to Tail, let code placement work it out later.
575  DEBUG(dbgs() << "Converting to unconditional branch.\n");
577  TII->insertBranch(*Head, Tail, nullptr, EmptyCond, HeadDL);
578  Head->addSuccessor(Tail);
579  }
580  DEBUG(dbgs() << *Head);
581 }
582 
583 
584 //===----------------------------------------------------------------------===//
585 // EarlyIfConverter Pass
586 //===----------------------------------------------------------------------===//
587 
588 namespace {
589 class EarlyIfConverter : public MachineFunctionPass {
590  const TargetInstrInfo *TII;
591  const TargetRegisterInfo *TRI;
592  MCSchedModel SchedModel;
594  MachineDominatorTree *DomTree;
596  MachineTraceMetrics *Traces;
598  SSAIfConv IfConv;
599 
600 public:
601  static char ID;
602  EarlyIfConverter() : MachineFunctionPass(ID) {}
603  void getAnalysisUsage(AnalysisUsage &AU) const override;
604  bool runOnMachineFunction(MachineFunction &MF) override;
605  StringRef getPassName() const override { return "Early If-Conversion"; }
606 
607 private:
608  bool tryConvertIf(MachineBasicBlock*);
609  void updateDomTree(ArrayRef<MachineBasicBlock*> Removed);
610  void updateLoops(ArrayRef<MachineBasicBlock*> Removed);
611  void invalidateTraces();
612  bool shouldConvertIf();
613 };
614 } // end anonymous namespace
615 
616 char EarlyIfConverter::ID = 0;
618 
619 INITIALIZE_PASS_BEGIN(EarlyIfConverter,
620  "early-ifcvt", "Early If Converter", false, false)
624 INITIALIZE_PASS_END(EarlyIfConverter,
625  "early-ifcvt", "Early If Converter", false, false)
626 
627 void EarlyIfConverter::getAnalysisUsage(AnalysisUsage &AU) const {
628  AU.addRequired<MachineBranchProbabilityInfo>();
629  AU.addRequired<MachineDominatorTree>();
630  AU.addPreserved<MachineDominatorTree>();
631  AU.addRequired<MachineLoopInfo>();
632  AU.addPreserved<MachineLoopInfo>();
633  AU.addRequired<MachineTraceMetrics>();
634  AU.addPreserved<MachineTraceMetrics>();
636 }
637 
638 /// Update the dominator tree after if-conversion erased some blocks.
639 void EarlyIfConverter::updateDomTree(ArrayRef<MachineBasicBlock*> Removed) {
640  // convertIf can remove TBB, FBB, and Tail can be merged into Head.
641  // TBB and FBB should not dominate any blocks.
642  // Tail children should be transferred to Head.
643  MachineDomTreeNode *HeadNode = DomTree->getNode(IfConv.Head);
644  for (unsigned i = 0, e = Removed.size(); i != e; ++i) {
645  MachineDomTreeNode *Node = DomTree->getNode(Removed[i]);
646  assert(Node != HeadNode && "Cannot erase the head node");
647  while (Node->getNumChildren()) {
648  assert(Node->getBlock() == IfConv.Tail && "Unexpected children");
649  DomTree->changeImmediateDominator(Node->getChildren().back(), HeadNode);
650  }
651  DomTree->eraseNode(Removed[i]);
652  }
653 }
654 
655 /// Update LoopInfo after if-conversion.
656 void EarlyIfConverter::updateLoops(ArrayRef<MachineBasicBlock*> Removed) {
657  if (!Loops)
658  return;
659  // If-conversion doesn't change loop structure, and it doesn't mess with back
660  // edges, so updating LoopInfo is simply removing the dead blocks.
661  for (unsigned i = 0, e = Removed.size(); i != e; ++i)
662  Loops->removeBlock(Removed[i]);
663 }
664 
665 /// Invalidate MachineTraceMetrics before if-conversion.
666 void EarlyIfConverter::invalidateTraces() {
667  Traces->verifyAnalysis();
668  Traces->invalidate(IfConv.Head);
669  Traces->invalidate(IfConv.Tail);
670  Traces->invalidate(IfConv.TBB);
671  Traces->invalidate(IfConv.FBB);
672  Traces->verifyAnalysis();
673 }
674 
675 // Adjust cycles with downward saturation.
676 static unsigned adjCycles(unsigned Cyc, int Delta) {
677  if (Delta < 0 && Cyc + Delta > Cyc)
678  return 0;
679  return Cyc + Delta;
680 }
681 
682 /// Apply cost model and heuristics to the if-conversion in IfConv.
683 /// Return true if the conversion is a good idea.
684 ///
685 bool EarlyIfConverter::shouldConvertIf() {
686  // Stress testing mode disables all cost considerations.
687  if (Stress)
688  return true;
689 
690  if (!MinInstr)
691  MinInstr = Traces->getEnsemble(MachineTraceMetrics::TS_MinInstrCount);
692 
693  MachineTraceMetrics::Trace TBBTrace = MinInstr->getTrace(IfConv.getTPred());
694  MachineTraceMetrics::Trace FBBTrace = MinInstr->getTrace(IfConv.getFPred());
695  DEBUG(dbgs() << "TBB: " << TBBTrace << "FBB: " << FBBTrace);
696  unsigned MinCrit = std::min(TBBTrace.getCriticalPath(),
697  FBBTrace.getCriticalPath());
698 
699  // Set a somewhat arbitrary limit on the critical path extension we accept.
700  unsigned CritLimit = SchedModel.MispredictPenalty/2;
701 
702  // If-conversion only makes sense when there is unexploited ILP. Compute the
703  // maximum-ILP resource length of the trace after if-conversion. Compare it
704  // to the shortest critical path.
706  if (IfConv.TBB != IfConv.Tail)
707  ExtraBlocks.push_back(IfConv.TBB);
708  unsigned ResLength = FBBTrace.getResourceLength(ExtraBlocks);
709  DEBUG(dbgs() << "Resource length " << ResLength
710  << ", minimal critical path " << MinCrit << '\n');
711  if (ResLength > MinCrit + CritLimit) {
712  DEBUG(dbgs() << "Not enough available ILP.\n");
713  return false;
714  }
715 
716  // Assume that the depth of the first head terminator will also be the depth
717  // of the select instruction inserted, as determined by the flag dependency.
718  // TBB / FBB data dependencies may delay the select even more.
719  MachineTraceMetrics::Trace HeadTrace = MinInstr->getTrace(IfConv.Head);
720  unsigned BranchDepth =
721  HeadTrace.getInstrCycles(*IfConv.Head->getFirstTerminator()).Depth;
722  DEBUG(dbgs() << "Branch depth: " << BranchDepth << '\n');
723 
724  // Look at all the tail phis, and compute the critical path extension caused
725  // by inserting select instructions.
726  MachineTraceMetrics::Trace TailTrace = MinInstr->getTrace(IfConv.Tail);
727  for (unsigned i = 0, e = IfConv.PHIs.size(); i != e; ++i) {
728  SSAIfConv::PHIInfo &PI = IfConv.PHIs[i];
729  unsigned Slack = TailTrace.getInstrSlack(*PI.PHI);
730  unsigned MaxDepth = Slack + TailTrace.getInstrCycles(*PI.PHI).Depth;
731  DEBUG(dbgs() << "Slack " << Slack << ":\t" << *PI.PHI);
732 
733  // The condition is pulled into the critical path.
734  unsigned CondDepth = adjCycles(BranchDepth, PI.CondCycles);
735  if (CondDepth > MaxDepth) {
736  unsigned Extra = CondDepth - MaxDepth;
737  DEBUG(dbgs() << "Condition adds " << Extra << " cycles.\n");
738  if (Extra > CritLimit) {
739  DEBUG(dbgs() << "Exceeds limit of " << CritLimit << '\n');
740  return false;
741  }
742  }
743 
744  // The TBB value is pulled into the critical path.
745  unsigned TDepth = adjCycles(TBBTrace.getPHIDepth(*PI.PHI), PI.TCycles);
746  if (TDepth > MaxDepth) {
747  unsigned Extra = TDepth - MaxDepth;
748  DEBUG(dbgs() << "TBB data adds " << Extra << " cycles.\n");
749  if (Extra > CritLimit) {
750  DEBUG(dbgs() << "Exceeds limit of " << CritLimit << '\n');
751  return false;
752  }
753  }
754 
755  // The FBB value is pulled into the critical path.
756  unsigned FDepth = adjCycles(FBBTrace.getPHIDepth(*PI.PHI), PI.FCycles);
757  if (FDepth > MaxDepth) {
758  unsigned Extra = FDepth - MaxDepth;
759  DEBUG(dbgs() << "FBB data adds " << Extra << " cycles.\n");
760  if (Extra > CritLimit) {
761  DEBUG(dbgs() << "Exceeds limit of " << CritLimit << '\n');
762  return false;
763  }
764  }
765  }
766  return true;
767 }
768 
769 /// Attempt repeated if-conversion on MBB, return true if successful.
770 ///
771 bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
772  bool Changed = false;
773  while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
774  // If-convert MBB and update analyses.
775  invalidateTraces();
777  IfConv.convertIf(RemovedBlocks);
778  Changed = true;
779  updateDomTree(RemovedBlocks);
780  updateLoops(RemovedBlocks);
781  }
782  return Changed;
783 }
784 
785 bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) {
786  DEBUG(dbgs() << "********** EARLY IF-CONVERSION **********\n"
787  << "********** Function: " << MF.getName() << '\n');
788  if (skipFunction(*MF.getFunction()))
789  return false;
790 
791  // Only run if conversion if the target wants it.
792  const TargetSubtargetInfo &STI = MF.getSubtarget();
793  if (!STI.enableEarlyIfConversion())
794  return false;
795 
796  TII = STI.getInstrInfo();
797  TRI = STI.getRegisterInfo();
798  SchedModel = STI.getSchedModel();
799  MRI = &MF.getRegInfo();
800  DomTree = &getAnalysis<MachineDominatorTree>();
801  Loops = getAnalysisIfAvailable<MachineLoopInfo>();
802  Traces = &getAnalysis<MachineTraceMetrics>();
803  MinInstr = nullptr;
804 
805  bool Changed = false;
806  IfConv.runOnMachineFunction(MF);
807 
808  // Visit blocks in dominator tree post-order. The post-order enables nested
809  // if-conversion in a single pass. The tryConvertIf() function may erase
810  // blocks, but only blocks dominated by the head block. This makes it safe to
811  // update the dominator tree while the post-order iterator is still active.
812  for (auto DomNode : post_order(DomTree))
813  if (tryConvertIf(DomNode->getBlock()))
814  Changed = true;
815 
816  return Changed;
817 }
unsigned succ_size() const
void push_back(const T &Elt)
Definition: SmallVector.h:211
early ifcvt
static cl::opt< bool > Stress("stress-early-ifcvt", cl::Hidden, cl::desc("Turn all knobs to 11"))
const std::vector< DomTreeNodeBase< NodeT > * > & getChildren() const
STATISTIC(NumFunctions,"Total number of functions")
size_t i
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned Depth
Earliest issue cycle as determined by data dependencies and instruction latencies from the beginning ...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned getCriticalPath() const
Return the length of the (data dependency) critical path through the trace.
char & EarlyIfConverterID
EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions...
A debug info location.
Definition: DebugLoc.h:34
A trace ensemble is a collection of traces selected using the same strategy, for example 'minimum res...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
static unsigned InstrCount
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:440
InstrCycles getInstrCycles(const MachineInstr &MI) const
Return the depth and height of MI.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
Hexagon Hardware Loops
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
Reg
All possible values of the reg field in the ModR/M byte.
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
Select the trace through a block that has the fewest instructions.
MachineBasicBlock * MBB
Base class for the actual dominator tree node.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
NodeT * getBlock() const
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:131
TargetInstrInfo - Interface to description of machine instruction set.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned getResourceLength(ArrayRef< const MachineBasicBlock * > Extrablocks=None, ArrayRef< const MCSchedClassDesc * > ExtraInstrs=None, ArrayRef< const MCSchedClassDesc * > RemoveInstrs=None) const
Return the resource length of the trace.
virtual bool enableEarlyIfConversion() const
Enable the use of the early if conversion pass.
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
iterator_range< po_iterator< T > > post_order(const T &G)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A trace represents a plausible sequence of executed basic blocks that passes through the current basi...
const unsigned MaxDepth
Printable PrintRegUnit(unsigned Unit, const TargetRegisterInfo *TRI)
Create Printable object to print register units on a raw_ostream.
static unsigned adjCycles(unsigned Cyc, int Delta)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
unsigned getPHIDepth(const MachineInstr &PHI) const
Return the Depth of a PHI instruction in a trace center block successor.
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static cl::opt< unsigned > BlockInstrLimit("early-ifcvt-limit", cl::init(30), cl::Hidden, cl::desc("Maximum number of instructions per speculated block."))
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
early Early If false
INITIALIZE_PASS_BEGIN(EarlyIfConverter,"early-ifcvt","Early If Converter", false, false) INITIALIZE_PASS_END(EarlyIfConverter
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
Remove the branching code at the end of the specific MBB.
Maximum length of the test input If
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
unsigned getInstrSlack(const MachineInstr &MI) const
Return the slack of MI.
Representation of each machine instruction.
Definition: MachineInstr.h:52
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
#define I(x, y, z)
Definition: MD5.cpp:54
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual const TargetInstrInfo * getInstrInfo() const
#define DEBUG(X)
Definition: Debug.h:100
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Machine model for scheduling, bundling, and heuristics.
Definition: MCSchedule.h:136
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const
size_t getNumChildren() const
early Early If Converter