LLVM  3.7.0
BranchFolding.cpp
Go to the documentation of this file.
1 //===-- BranchFolding.cpp - Fold machine code branch instructions ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass forwards branches to unconditional branches to make them branch
11 // directly to the target block. This pass often results in dead MBB's, which
12 // it then removes.
13 //
14 // Note that this pass must be run after register allocation, it cannot handle
15 // SSA form.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "BranchFolding.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
30 #include "llvm/CodeGen/Passes.h"
32 #include "llvm/IR/Function.h"
34 #include "llvm/Support/Debug.h"
40 #include <algorithm>
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "branchfolding"
44 
45 STATISTIC(NumDeadBlocks, "Number of dead blocks removed");
46 STATISTIC(NumBranchOpts, "Number of branches optimized");
47 STATISTIC(NumTailMerge , "Number of block tails merged");
48 STATISTIC(NumHoist , "Number of times common instructions are hoisted");
49 
50 static cl::opt<cl::boolOrDefault> FlagEnableTailMerge("enable-tail-merge",
52 
53 // Throttle for huge numbers of predecessors (compile speed problems)
54 static cl::opt<unsigned>
55 TailMergeThreshold("tail-merge-threshold",
56  cl::desc("Max number of predecessors to consider tail merging"),
57  cl::init(150), cl::Hidden);
58 
59 // Heuristic for tail merging (and, inversely, tail duplication).
60 // TODO: This should be replaced with a target query.
61 static cl::opt<unsigned>
62 TailMergeSize("tail-merge-size",
63  cl::desc("Min number of instructions to consider tail merging"),
64  cl::init(3), cl::Hidden);
65 
66 namespace {
67  /// BranchFolderPass - Wrap branch folder in a machine function pass.
68  class BranchFolderPass : public MachineFunctionPass {
69  public:
70  static char ID;
71  explicit BranchFolderPass(): MachineFunctionPass(ID) {}
72 
73  bool runOnMachineFunction(MachineFunction &MF) override;
74 
75  void getAnalysisUsage(AnalysisUsage &AU) const override {
80  }
81  };
82 }
83 
84 char BranchFolderPass::ID = 0;
86 
87 INITIALIZE_PASS(BranchFolderPass, "branch-folder",
88  "Control Flow Optimizer", false, false)
89 
90 bool BranchFolderPass::runOnMachineFunction(MachineFunction &MF) {
91  if (skipOptnoneFunction(*MF.getFunction()))
92  return false;
93 
94  TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>();
95  // TailMerge can create jump into if branches that make CFG irreducible for
96  // HW that requires structurized CFG.
97  bool EnableTailMerge = !MF.getTarget().requiresStructuredCFG() &&
98  PassConfig->getEnableTailMerge();
99  BranchFolder Folder(EnableTailMerge, /*CommonHoist=*/true,
100  getAnalysis<MachineBlockFrequencyInfo>(),
101  getAnalysis<MachineBranchProbabilityInfo>());
102  return Folder.OptimizeFunction(MF, MF.getSubtarget().getInstrInfo(),
103  MF.getSubtarget().getRegisterInfo(),
104  getAnalysisIfAvailable<MachineModuleInfo>());
105 }
106 
107 BranchFolder::BranchFolder(bool defaultEnableTailMerge, bool CommonHoist,
108  const MachineBlockFrequencyInfo &FreqInfo,
109  const MachineBranchProbabilityInfo &ProbInfo)
110  : EnableHoistCommonCode(CommonHoist), MBBFreqInfo(FreqInfo),
111  MBPI(ProbInfo) {
112  switch (FlagEnableTailMerge) {
113  case cl::BOU_UNSET: EnableTailMerge = defaultEnableTailMerge; break;
114  case cl::BOU_TRUE: EnableTailMerge = true; break;
115  case cl::BOU_FALSE: EnableTailMerge = false; break;
116  }
117 }
118 
119 /// RemoveDeadBlock - Remove the specified dead machine basic block from the
120 /// function, updating the CFG.
121 void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
122  assert(MBB->pred_empty() && "MBB must be dead!");
123  DEBUG(dbgs() << "\nRemoving MBB: " << *MBB);
124 
125  MachineFunction *MF = MBB->getParent();
126  // drop all successors.
127  while (!MBB->succ_empty())
128  MBB->removeSuccessor(MBB->succ_end()-1);
129 
130  // Avoid matching if this pointer gets reused.
131  TriedMerging.erase(MBB);
132 
133  // Remove the block.
134  MF->erase(MBB);
135 }
136 
137 /// OptimizeImpDefsBlock - If a basic block is just a bunch of implicit_def
138 /// followed by terminators, and if the implicitly defined registers are not
139 /// used by the terminators, remove those implicit_def's. e.g.
140 /// BB1:
141 /// r0 = implicit_def
142 /// r1 = implicit_def
143 /// br
144 /// This block can be optimized away later if the implicit instructions are
145 /// removed.
146 bool BranchFolder::OptimizeImpDefsBlock(MachineBasicBlock *MBB) {
147  SmallSet<unsigned, 4> ImpDefRegs;
149  while (I != MBB->end()) {
150  if (!I->isImplicitDef())
151  break;
152  unsigned Reg = I->getOperand(0).getReg();
153  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
154  SubRegs.isValid(); ++SubRegs)
155  ImpDefRegs.insert(*SubRegs);
156  ++I;
157  }
158  if (ImpDefRegs.empty())
159  return false;
160 
161  MachineBasicBlock::iterator FirstTerm = I;
162  while (I != MBB->end()) {
163  if (!TII->isUnpredicatedTerminator(I))
164  return false;
165  // See if it uses any of the implicitly defined registers.
166  for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
167  MachineOperand &MO = I->getOperand(i);
168  if (!MO.isReg() || !MO.isUse())
169  continue;
170  unsigned Reg = MO.getReg();
171  if (ImpDefRegs.count(Reg))
172  return false;
173  }
174  ++I;
175  }
176 
177  I = MBB->begin();
178  while (I != FirstTerm) {
179  MachineInstr *ImpDefMI = &*I;
180  ++I;
181  MBB->erase(ImpDefMI);
182  }
183 
184  return true;
185 }
186 
187 /// OptimizeFunction - Perhaps branch folding, tail merging and other
188 /// CFG optimizations on the given function.
190  const TargetInstrInfo *tii,
191  const TargetRegisterInfo *tri,
192  MachineModuleInfo *mmi) {
193  if (!tii) return false;
194 
195  TriedMerging.clear();
196 
197  TII = tii;
198  TRI = tri;
199  MMI = mmi;
200  RS = nullptr;
201 
202  // Use a RegScavenger to help update liveness when required.
203  MachineRegisterInfo &MRI = MF.getRegInfo();
204  if (MRI.tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF))
205  RS = new RegScavenger();
206  else
207  MRI.invalidateLiveness();
208 
209  // Fix CFG. The later algorithms expect it to be right.
210  bool MadeChange = false;
211  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; I++) {
212  MachineBasicBlock *MBB = I, *TBB = nullptr, *FBB = nullptr;
214  if (!TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, true))
215  MadeChange |= MBB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty());
216  MadeChange |= OptimizeImpDefsBlock(MBB);
217  }
218 
219  bool MadeChangeThisIteration = true;
220  while (MadeChangeThisIteration) {
221  MadeChangeThisIteration = TailMergeBlocks(MF);
222  MadeChangeThisIteration |= OptimizeBranches(MF);
223  if (EnableHoistCommonCode)
224  MadeChangeThisIteration |= HoistCommonCode(MF);
225  MadeChange |= MadeChangeThisIteration;
226  }
227 
228  // See if any jump tables have become dead as the code generator
229  // did its thing.
231  if (!JTI) {
232  delete RS;
233  return MadeChange;
234  }
235 
236  // Walk the function to find jump tables that are live.
237  BitVector JTIsLive(JTI->getJumpTables().size());
238  for (MachineFunction::iterator BB = MF.begin(), E = MF.end();
239  BB != E; ++BB) {
240  for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end();
241  I != E; ++I)
242  for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) {
243  MachineOperand &Op = I->getOperand(op);
244  if (!Op.isJTI()) continue;
245 
246  // Remember that this JT is live.
247  JTIsLive.set(Op.getIndex());
248  }
249  }
250 
251  // Finally, remove dead jump tables. This happens when the
252  // indirect jump was unreachable (and thus deleted).
253  for (unsigned i = 0, e = JTIsLive.size(); i != e; ++i)
254  if (!JTIsLive.test(i)) {
255  JTI->RemoveJumpTable(i);
256  MadeChange = true;
257  }
258 
259  delete RS;
260  return MadeChange;
261 }
262 
263 //===----------------------------------------------------------------------===//
264 // Tail Merging of Blocks
265 //===----------------------------------------------------------------------===//
266 
267 /// HashMachineInstr - Compute a hash value for MI and its operands.
268 static unsigned HashMachineInstr(const MachineInstr *MI) {
269  unsigned Hash = MI->getOpcode();
270  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
271  const MachineOperand &Op = MI->getOperand(i);
272 
273  // Merge in bits from the operand if easy. We can't use MachineOperand's
274  // hash_code here because it's not deterministic and we sort by hash value
275  // later.
276  unsigned OperandHash = 0;
277  switch (Op.getType()) {
279  OperandHash = Op.getReg();
280  break;
282  OperandHash = Op.getImm();
283  break;
285  OperandHash = Op.getMBB()->getNumber();
286  break;
290  OperandHash = Op.getIndex();
291  break;
294  // Global address / external symbol are too hard, don't bother, but do
295  // pull in the offset.
296  OperandHash = Op.getOffset();
297  break;
298  default:
299  break;
300  }
301 
302  Hash += ((OperandHash << 3) | Op.getType()) << (i & 31);
303  }
304  return Hash;
305 }
306 
307 /// HashEndOfMBB - Hash the last instruction in the MBB.
308 static unsigned HashEndOfMBB(const MachineBasicBlock *MBB) {
310  if (I == MBB->end())
311  return 0;
312 
313  return HashMachineInstr(I);
314 }
315 
316 /// ComputeCommonTailLength - Given two machine basic blocks, compute the number
317 /// of instructions they actually have in common together at their end. Return
318 /// iterators for the first shared instruction in each block.
320  MachineBasicBlock *MBB2,
323  I1 = MBB1->end();
324  I2 = MBB2->end();
325 
326  unsigned TailLen = 0;
327  while (I1 != MBB1->begin() && I2 != MBB2->begin()) {
328  --I1; --I2;
329  // Skip debugging pseudos; necessary to avoid changing the code.
330  while (I1->isDebugValue()) {
331  if (I1==MBB1->begin()) {
332  while (I2->isDebugValue()) {
333  if (I2==MBB2->begin())
334  // I1==DBG at begin; I2==DBG at begin
335  return TailLen;
336  --I2;
337  }
338  ++I2;
339  // I1==DBG at begin; I2==non-DBG, or first of DBGs not at begin
340  return TailLen;
341  }
342  --I1;
343  }
344  // I1==first (untested) non-DBG preceding known match
345  while (I2->isDebugValue()) {
346  if (I2==MBB2->begin()) {
347  ++I1;
348  // I1==non-DBG, or first of DBGs not at begin; I2==DBG at begin
349  return TailLen;
350  }
351  --I2;
352  }
353  // I1, I2==first (untested) non-DBGs preceding known match
354  if (!I1->isIdenticalTo(I2) ||
355  // FIXME: This check is dubious. It's used to get around a problem where
356  // people incorrectly expect inline asm directives to remain in the same
357  // relative order. This is untenable because normal compiler
358  // optimizations (like this one) may reorder and/or merge these
359  // directives.
360  I1->isInlineAsm()) {
361  ++I1; ++I2;
362  break;
363  }
364  ++TailLen;
365  }
366  // Back past possible debugging pseudos at beginning of block. This matters
367  // when one block differs from the other only by whether debugging pseudos
368  // are present at the beginning. (This way, the various checks later for
369  // I1==MBB1->begin() work as expected.)
370  if (I1 == MBB1->begin() && I2 != MBB2->begin()) {
371  --I2;
372  while (I2->isDebugValue()) {
373  if (I2 == MBB2->begin())
374  return TailLen;
375  --I2;
376  }
377  ++I2;
378  }
379  if (I2 == MBB2->begin() && I1 != MBB1->begin()) {
380  --I1;
381  while (I1->isDebugValue()) {
382  if (I1 == MBB1->begin())
383  return TailLen;
384  --I1;
385  }
386  ++I1;
387  }
388  return TailLen;
389 }
390 
391 void BranchFolder::MaintainLiveIns(MachineBasicBlock *CurMBB,
392  MachineBasicBlock *NewMBB) {
393  if (RS) {
394  RS->enterBasicBlock(CurMBB);
395  if (!CurMBB->empty())
396  RS->forward(std::prev(CurMBB->end()));
397  for (unsigned int i = 1, e = TRI->getNumRegs(); i != e; i++)
398  if (RS->isRegUsed(i, false))
399  NewMBB->addLiveIn(i);
400  }
401 }
402 
403 /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
404 /// after it, replacing it with an unconditional branch to NewDest.
405 void BranchFolder::ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
406  MachineBasicBlock *NewDest) {
407  MachineBasicBlock *CurMBB = OldInst->getParent();
408 
409  TII->ReplaceTailWithBranchTo(OldInst, NewDest);
410 
411  // For targets that use the register scavenger, we must maintain LiveIns.
412  MaintainLiveIns(CurMBB, NewDest);
413 
414  ++NumTailMerge;
415 }
416 
417 /// SplitMBBAt - Given a machine basic block and an iterator into it, split the
418 /// MBB so that the part before the iterator falls into the part starting at the
419 /// iterator. This returns the new MBB.
420 MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB,
422  const BasicBlock *BB) {
423  if (!TII->isLegalToSplitMBBAt(CurMBB, BBI1))
424  return nullptr;
425 
426  MachineFunction &MF = *CurMBB.getParent();
427 
428  // Create the fall-through block.
429  MachineFunction::iterator MBBI = &CurMBB;
431  CurMBB.getParent()->insert(++MBBI, NewMBB);
432 
433  // Move all the successors of this block to the specified block.
434  NewMBB->transferSuccessors(&CurMBB);
435 
436  // Add an edge from CurMBB to NewMBB for the fall-through.
437  CurMBB.addSuccessor(NewMBB);
438 
439  // Splice the code over.
440  NewMBB->splice(NewMBB->end(), &CurMBB, BBI1, CurMBB.end());
441 
442  // NewMBB inherits CurMBB's block frequency.
443  MBBFreqInfo.setBlockFreq(NewMBB, MBBFreqInfo.getBlockFreq(&CurMBB));
444 
445  // For targets that use the register scavenger, we must maintain LiveIns.
446  MaintainLiveIns(&CurMBB, NewMBB);
447 
448  return NewMBB;
449 }
450 
451 /// EstimateRuntime - Make a rough estimate for how long it will take to run
452 /// the specified code.
455  unsigned Time = 0;
456  for (; I != E; ++I) {
457  if (I->isDebugValue())
458  continue;
459  if (I->isCall())
460  Time += 10;
461  else if (I->mayLoad() || I->mayStore())
462  Time += 2;
463  else
464  ++Time;
465  }
466  return Time;
467 }
468 
469 // CurMBB needs to add an unconditional branch to SuccMBB (we removed these
470 // branches temporarily for tail merging). In the case where CurMBB ends
471 // with a conditional branch to the next block, optimize by reversing the
472 // test and conditionally branching to SuccMBB instead.
473 static void FixTail(MachineBasicBlock *CurMBB, MachineBasicBlock *SuccBB,
474  const TargetInstrInfo *TII) {
475  MachineFunction *MF = CurMBB->getParent();
477  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
479  DebugLoc dl; // FIXME: this is nowhere
480  if (I != MF->end() &&
481  !TII->AnalyzeBranch(*CurMBB, TBB, FBB, Cond, true)) {
482  MachineBasicBlock *NextBB = I;
483  if (TBB == NextBB && !Cond.empty() && !FBB) {
484  if (!TII->ReverseBranchCondition(Cond)) {
485  TII->RemoveBranch(*CurMBB);
486  TII->InsertBranch(*CurMBB, SuccBB, nullptr, Cond, dl);
487  return;
488  }
489  }
490  }
491  TII->InsertBranch(*CurMBB, SuccBB, nullptr,
493 }
494 
495 bool
496 BranchFolder::MergePotentialsElt::operator<(const MergePotentialsElt &o) const {
497  if (getHash() < o.getHash())
498  return true;
499  if (getHash() > o.getHash())
500  return false;
501  if (getBlock()->getNumber() < o.getBlock()->getNumber())
502  return true;
503  if (getBlock()->getNumber() > o.getBlock()->getNumber())
504  return false;
505  // _GLIBCXX_DEBUG checks strict weak ordering, which involves comparing
506  // an object with itself.
507 #ifndef _GLIBCXX_DEBUG
508  llvm_unreachable("Predecessor appears twice");
509 #else
510  return false;
511 #endif
512 }
513 
515 BranchFolder::MBFIWrapper::getBlockFreq(const MachineBasicBlock *MBB) const {
516  auto I = MergedBBFreq.find(MBB);
517 
518  if (I != MergedBBFreq.end())
519  return I->second;
520 
521  return MBFI.getBlockFreq(MBB);
522 }
523 
524 void BranchFolder::MBFIWrapper::setBlockFreq(const MachineBasicBlock *MBB,
525  BlockFrequency F) {
526  MergedBBFreq[MBB] = F;
527 }
528 
529 /// CountTerminators - Count the number of terminators in the given
530 /// block and set I to the position of the first non-terminator, if there
531 /// is one, or MBB->end() otherwise.
532 static unsigned CountTerminators(MachineBasicBlock *MBB,
534  I = MBB->end();
535  unsigned NumTerms = 0;
536  for (;;) {
537  if (I == MBB->begin()) {
538  I = MBB->end();
539  break;
540  }
541  --I;
542  if (!I->isTerminator()) break;
543  ++NumTerms;
544  }
545  return NumTerms;
546 }
547 
548 /// ProfitableToMerge - Check if two machine basic blocks have a common tail
549 /// and decide if it would be profitable to merge those tails. Return the
550 /// length of the common tail and iterators to the first common instruction
551 /// in each block.
553  MachineBasicBlock *MBB2,
554  unsigned minCommonTailLength,
555  unsigned &CommonTailLen,
558  MachineBasicBlock *SuccBB,
559  MachineBasicBlock *PredBB) {
560  CommonTailLen = ComputeCommonTailLength(MBB1, MBB2, I1, I2);
561  if (CommonTailLen == 0)
562  return false;
563  DEBUG(dbgs() << "Common tail length of BB#" << MBB1->getNumber()
564  << " and BB#" << MBB2->getNumber() << " is " << CommonTailLen
565  << '\n');
566 
567  // It's almost always profitable to merge any number of non-terminator
568  // instructions with the block that falls through into the common successor.
569  if (MBB1 == PredBB || MBB2 == PredBB) {
571  unsigned NumTerms = CountTerminators(MBB1 == PredBB ? MBB2 : MBB1, I);
572  if (CommonTailLen > NumTerms)
573  return true;
574  }
575 
576  // If one of the blocks can be completely merged and happens to be in
577  // a position where the other could fall through into it, merge any number
578  // of instructions, because it can be done without a branch.
579  // TODO: If the blocks are not adjacent, move one of them so that they are?
580  if (MBB1->isLayoutSuccessor(MBB2) && I2 == MBB2->begin())
581  return true;
582  if (MBB2->isLayoutSuccessor(MBB1) && I1 == MBB1->begin())
583  return true;
584 
585  // If both blocks have an unconditional branch temporarily stripped out,
586  // count that as an additional common instruction for the following
587  // heuristics.
588  unsigned EffectiveTailLen = CommonTailLen;
589  if (SuccBB && MBB1 != PredBB && MBB2 != PredBB &&
590  !MBB1->back().isBarrier() &&
591  !MBB2->back().isBarrier())
592  ++EffectiveTailLen;
593 
594  // Check if the common tail is long enough to be worthwhile.
595  if (EffectiveTailLen >= minCommonTailLength)
596  return true;
597 
598  // If we are optimizing for code size, 2 instructions in common is enough if
599  // we don't have to split a block. At worst we will be introducing 1 new
600  // branch instruction, which is likely to be smaller than the 2
601  // instructions that would be deleted in the merge.
602  MachineFunction *MF = MBB1->getParent();
603  if (EffectiveTailLen >= 2 &&
605  (I1 == MBB1->begin() || I2 == MBB2->begin()))
606  return true;
607 
608  return false;
609 }
610 
611 /// ComputeSameTails - Look through all the blocks in MergePotentials that have
612 /// hash CurHash (guaranteed to match the last element). Build the vector
613 /// SameTails of all those that have the (same) largest number of instructions
614 /// in common of any pair of these blocks. SameTails entries contain an
615 /// iterator into MergePotentials (from which the MachineBasicBlock can be
616 /// found) and a MachineBasicBlock::iterator into that MBB indicating the
617 /// instruction where the matching code sequence begins.
618 /// Order of elements in SameTails is the reverse of the order in which
619 /// those blocks appear in MergePotentials (where they are not necessarily
620 /// consecutive).
621 unsigned BranchFolder::ComputeSameTails(unsigned CurHash,
622  unsigned minCommonTailLength,
623  MachineBasicBlock *SuccBB,
624  MachineBasicBlock *PredBB) {
625  unsigned maxCommonTailLength = 0U;
626  SameTails.clear();
627  MachineBasicBlock::iterator TrialBBI1, TrialBBI2;
628  MPIterator HighestMPIter = std::prev(MergePotentials.end());
629  for (MPIterator CurMPIter = std::prev(MergePotentials.end()),
630  B = MergePotentials.begin();
631  CurMPIter != B && CurMPIter->getHash() == CurHash; --CurMPIter) {
632  for (MPIterator I = std::prev(CurMPIter); I->getHash() == CurHash; --I) {
633  unsigned CommonTailLen;
634  if (ProfitableToMerge(CurMPIter->getBlock(), I->getBlock(),
635  minCommonTailLength,
636  CommonTailLen, TrialBBI1, TrialBBI2,
637  SuccBB, PredBB)) {
638  if (CommonTailLen > maxCommonTailLength) {
639  SameTails.clear();
640  maxCommonTailLength = CommonTailLen;
641  HighestMPIter = CurMPIter;
642  SameTails.push_back(SameTailElt(CurMPIter, TrialBBI1));
643  }
644  if (HighestMPIter == CurMPIter &&
645  CommonTailLen == maxCommonTailLength)
646  SameTails.push_back(SameTailElt(I, TrialBBI2));
647  }
648  if (I == B)
649  break;
650  }
651  }
652  return maxCommonTailLength;
653 }
654 
655 /// RemoveBlocksWithHash - Remove all blocks with hash CurHash from
656 /// MergePotentials, restoring branches at ends of blocks as appropriate.
657 void BranchFolder::RemoveBlocksWithHash(unsigned CurHash,
658  MachineBasicBlock *SuccBB,
659  MachineBasicBlock *PredBB) {
660  MPIterator CurMPIter, B;
661  for (CurMPIter = std::prev(MergePotentials.end()),
662  B = MergePotentials.begin();
663  CurMPIter->getHash() == CurHash; --CurMPIter) {
664  // Put the unconditional branch back, if we need one.
665  MachineBasicBlock *CurMBB = CurMPIter->getBlock();
666  if (SuccBB && CurMBB != PredBB)
667  FixTail(CurMBB, SuccBB, TII);
668  if (CurMPIter == B)
669  break;
670  }
671  if (CurMPIter->getHash() != CurHash)
672  CurMPIter++;
673  MergePotentials.erase(CurMPIter, MergePotentials.end());
674 }
675 
676 /// CreateCommonTailOnlyBlock - None of the blocks to be tail-merged consist
677 /// only of the common tail. Create a block that does by splitting one.
678 bool BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB,
679  MachineBasicBlock *SuccBB,
680  unsigned maxCommonTailLength,
681  unsigned &commonTailIndex) {
682  commonTailIndex = 0;
683  unsigned TimeEstimate = ~0U;
684  for (unsigned i = 0, e = SameTails.size(); i != e; ++i) {
685  // Use PredBB if possible; that doesn't require a new branch.
686  if (SameTails[i].getBlock() == PredBB) {
687  commonTailIndex = i;
688  break;
689  }
690  // Otherwise, make a (fairly bogus) choice based on estimate of
691  // how long it will take the various blocks to execute.
692  unsigned t = EstimateRuntime(SameTails[i].getBlock()->begin(),
693  SameTails[i].getTailStartPos());
694  if (t <= TimeEstimate) {
695  TimeEstimate = t;
696  commonTailIndex = i;
697  }
698  }
699 
701  SameTails[commonTailIndex].getTailStartPos();
702  MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
703 
704  // If the common tail includes any debug info we will take it pretty
705  // randomly from one of the inputs. Might be better to remove it?
706  DEBUG(dbgs() << "\nSplitting BB#" << MBB->getNumber() << ", size "
707  << maxCommonTailLength);
708 
709  // If the split block unconditionally falls-thru to SuccBB, it will be
710  // merged. In control flow terms it should then take SuccBB's name. e.g. If
711  // SuccBB is an inner loop, the common tail is still part of the inner loop.
712  const BasicBlock *BB = (SuccBB && MBB->succ_size() == 1) ?
713  SuccBB->getBasicBlock() : MBB->getBasicBlock();
714  MachineBasicBlock *newMBB = SplitMBBAt(*MBB, BBI, BB);
715  if (!newMBB) {
716  DEBUG(dbgs() << "... failed!");
717  return false;
718  }
719 
720  SameTails[commonTailIndex].setBlock(newMBB);
721  SameTails[commonTailIndex].setTailStartPos(newMBB->begin());
722 
723  // If we split PredBB, newMBB is the new predecessor.
724  if (PredBB == MBB)
725  PredBB = newMBB;
726 
727  return true;
728 }
729 
730 static bool hasIdenticalMMOs(const MachineInstr *MI1, const MachineInstr *MI2) {
731  auto I1 = MI1->memoperands_begin(), E1 = MI1->memoperands_end();
732  auto I2 = MI2->memoperands_begin(), E2 = MI2->memoperands_end();
733  if ((E1 - I1) != (E2 - I2))
734  return false;
735  for (; I1 != E1; ++I1, ++I2) {
736  if (**I1 != **I2)
737  return false;
738  }
739  return true;
740 }
741 
742 static void
744  MachineBasicBlock &MBBCommon) {
745  // Remove MMOs from memory operations in the common block
746  // when they do not match the ones from the block being tail-merged.
747  // This ensures later passes conservatively compute dependencies.
748  MachineBasicBlock *MBB = MBBIStartPos->getParent();
749  // Note CommonTailLen does not necessarily matches the size of
750  // the common BB nor all its instructions because of debug
751  // instructions differences.
752  unsigned CommonTailLen = 0;
753  for (auto E = MBB->end(); MBBIStartPos != E; ++MBBIStartPos)
754  ++CommonTailLen;
755 
758  MachineBasicBlock::reverse_iterator MBBICommon = MBBCommon.rbegin();
759  MachineBasicBlock::reverse_iterator MBBIECommon = MBBCommon.rend();
760 
761  while (CommonTailLen--) {
762  assert(MBBI != MBBIE && "Reached BB end within common tail length!");
763  (void)MBBIE;
764 
765  if (MBBI->isDebugValue()) {
766  ++MBBI;
767  continue;
768  }
769 
770  while ((MBBICommon != MBBIECommon) && MBBICommon->isDebugValue())
771  ++MBBICommon;
772 
773  assert(MBBICommon != MBBIECommon &&
774  "Reached BB end within common tail length!");
775  assert(MBBICommon->isIdenticalTo(&*MBBI) && "Expected matching MIIs!");
776 
777  if (MBBICommon->mayLoad() || MBBICommon->mayStore())
778  if (!hasIdenticalMMOs(&*MBBI, &*MBBICommon))
779  MBBICommon->clearMemRefs();
780 
781  ++MBBI;
782  ++MBBICommon;
783  }
784 }
785 
786 // See if any of the blocks in MergePotentials (which all have a common single
787 // successor, or all have no successor) can be tail-merged. If there is a
788 // successor, any blocks in MergePotentials that are not tail-merged and
789 // are not immediately before Succ must have an unconditional branch to
790 // Succ added (but the predecessor/successor lists need no adjustment).
791 // The lone predecessor of Succ that falls through into Succ,
792 // if any, is given in PredBB.
793 
794 bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
795  MachineBasicBlock *PredBB) {
796  bool MadeChange = false;
797 
798  // Except for the special cases below, tail-merge if there are at least
799  // this many instructions in common.
800  unsigned minCommonTailLength = TailMergeSize;
801 
802  DEBUG(dbgs() << "\nTryTailMergeBlocks: ";
803  for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i)
804  dbgs() << "BB#" << MergePotentials[i].getBlock()->getNumber()
805  << (i == e-1 ? "" : ", ");
806  dbgs() << "\n";
807  if (SuccBB) {
808  dbgs() << " with successor BB#" << SuccBB->getNumber() << '\n';
809  if (PredBB)
810  dbgs() << " which has fall-through from BB#"
811  << PredBB->getNumber() << "\n";
812  }
813  dbgs() << "Looking for common tails of at least "
814  << minCommonTailLength << " instruction"
815  << (minCommonTailLength == 1 ? "" : "s") << '\n';
816  );
817 
818  // Sort by hash value so that blocks with identical end sequences sort
819  // together.
820  array_pod_sort(MergePotentials.begin(), MergePotentials.end());
821 
822  // Walk through equivalence sets looking for actual exact matches.
823  while (MergePotentials.size() > 1) {
824  unsigned CurHash = MergePotentials.back().getHash();
825 
826  // Build SameTails, identifying the set of blocks with this hash code
827  // and with the maximum number of instructions in common.
828  unsigned maxCommonTailLength = ComputeSameTails(CurHash,
829  minCommonTailLength,
830  SuccBB, PredBB);
831 
832  // If we didn't find any pair that has at least minCommonTailLength
833  // instructions in common, remove all blocks with this hash code and retry.
834  if (SameTails.empty()) {
835  RemoveBlocksWithHash(CurHash, SuccBB, PredBB);
836  continue;
837  }
838 
839  // If one of the blocks is the entire common tail (and not the entry
840  // block, which we can't jump to), we can treat all blocks with this same
841  // tail at once. Use PredBB if that is one of the possibilities, as that
842  // will not introduce any extra branches.
843  MachineBasicBlock *EntryBB = MergePotentials.begin()->getBlock()->
844  getParent()->begin();
845  unsigned commonTailIndex = SameTails.size();
846  // If there are two blocks, check to see if one can be made to fall through
847  // into the other.
848  if (SameTails.size() == 2 &&
849  SameTails[0].getBlock()->isLayoutSuccessor(SameTails[1].getBlock()) &&
850  SameTails[1].tailIsWholeBlock())
851  commonTailIndex = 1;
852  else if (SameTails.size() == 2 &&
853  SameTails[1].getBlock()->isLayoutSuccessor(
854  SameTails[0].getBlock()) &&
855  SameTails[0].tailIsWholeBlock())
856  commonTailIndex = 0;
857  else {
858  // Otherwise just pick one, favoring the fall-through predecessor if
859  // there is one.
860  for (unsigned i = 0, e = SameTails.size(); i != e; ++i) {
861  MachineBasicBlock *MBB = SameTails[i].getBlock();
862  if (MBB == EntryBB && SameTails[i].tailIsWholeBlock())
863  continue;
864  if (MBB == PredBB) {
865  commonTailIndex = i;
866  break;
867  }
868  if (SameTails[i].tailIsWholeBlock())
869  commonTailIndex = i;
870  }
871  }
872 
873  if (commonTailIndex == SameTails.size() ||
874  (SameTails[commonTailIndex].getBlock() == PredBB &&
875  !SameTails[commonTailIndex].tailIsWholeBlock())) {
876  // None of the blocks consist entirely of the common tail.
877  // Split a block so that one does.
878  if (!CreateCommonTailOnlyBlock(PredBB, SuccBB,
879  maxCommonTailLength, commonTailIndex)) {
880  RemoveBlocksWithHash(CurHash, SuccBB, PredBB);
881  continue;
882  }
883  }
884 
885  MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
886 
887  // Recompute commont tail MBB's edge weights and block frequency.
888  setCommonTailEdgeWeights(*MBB);
889 
890  // MBB is common tail. Adjust all other BB's to jump to this one.
891  // Traversal must be forwards so erases work.
892  DEBUG(dbgs() << "\nUsing common tail in BB#" << MBB->getNumber()
893  << " for ");
894  for (unsigned int i=0, e = SameTails.size(); i != e; ++i) {
895  if (commonTailIndex == i)
896  continue;
897  DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber()
898  << (i == e-1 ? "" : ", "));
899  // Remove MMOs from memory operations as needed.
900  removeMMOsFromMemoryOperations(SameTails[i].getTailStartPos(), *MBB);
901  // Hack the end off BB i, making it jump to BB commonTailIndex instead.
902  ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB);
903  // BB i is no longer a predecessor of SuccBB; remove it from the worklist.
904  MergePotentials.erase(SameTails[i].getMPIter());
905  }
906  DEBUG(dbgs() << "\n");
907  // We leave commonTailIndex in the worklist in case there are other blocks
908  // that match it with a smaller number of instructions.
909  MadeChange = true;
910  }
911  return MadeChange;
912 }
913 
914 bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
915  bool MadeChange = false;
916  if (!EnableTailMerge) return MadeChange;
917 
918  // First find blocks with no successors.
919  MergePotentials.clear();
920  for (MachineFunction::iterator I = MF.begin(), E = MF.end();
921  I != E && MergePotentials.size() < TailMergeThreshold; ++I) {
922  if (TriedMerging.count(I))
923  continue;
924  if (I->succ_empty())
925  MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(I), I));
926  }
927 
928  // If this is a large problem, avoid visiting the same basic blocks
929  // multiple times.
930  if (MergePotentials.size() == TailMergeThreshold)
931  for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i)
932  TriedMerging.insert(MergePotentials[i].getBlock());
933 
934  // See if we can do any tail merging on those.
935  if (MergePotentials.size() >= 2)
936  MadeChange |= TryTailMergeBlocks(nullptr, nullptr);
937 
938  // Look at blocks (IBB) with multiple predecessors (PBB).
939  // We change each predecessor to a canonical form, by
940  // (1) temporarily removing any unconditional branch from the predecessor
941  // to IBB, and
942  // (2) alter conditional branches so they branch to the other block
943  // not IBB; this may require adding back an unconditional branch to IBB
944  // later, where there wasn't one coming in. E.g.
945  // Bcc IBB
946  // fallthrough to QBB
947  // here becomes
948  // Bncc QBB
949  // with a conceptual B to IBB after that, which never actually exists.
950  // With those changes, we see whether the predecessors' tails match,
951  // and merge them if so. We change things out of canonical form and
952  // back to the way they were later in the process. (OptimizeBranches
953  // would undo some of this, but we can't use it, because we'd get into
954  // a compile-time infinite loop repeatedly doing and undoing the same
955  // transformations.)
956 
957  for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end();
958  I != E; ++I) {
959  if (I->pred_size() < 2) continue;
961  MachineBasicBlock *IBB = I;
962  MachineBasicBlock *PredBB = std::prev(I);
963  MergePotentials.clear();
964  for (MachineBasicBlock::pred_iterator P = I->pred_begin(),
965  E2 = I->pred_end();
966  P != E2 && MergePotentials.size() < TailMergeThreshold; ++P) {
967  MachineBasicBlock *PBB = *P;
968  if (TriedMerging.count(PBB))
969  continue;
970 
971  // Skip blocks that loop to themselves, can't tail merge these.
972  if (PBB == IBB)
973  continue;
974 
975  // Visit each predecessor only once.
976  if (!UniquePreds.insert(PBB).second)
977  continue;
978 
979  // Skip blocks which may jump to a landing pad. Can't tail merge these.
980  if (PBB->getLandingPadSuccessor())
981  continue;
982 
983  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
985  if (!TII->AnalyzeBranch(*PBB, TBB, FBB, Cond, true)) {
986  // Failing case: IBB is the target of a cbr, and we cannot reverse the
987  // branch.
988  SmallVector<MachineOperand, 4> NewCond(Cond);
989  if (!Cond.empty() && TBB == IBB) {
990  if (TII->ReverseBranchCondition(NewCond))
991  continue;
992  // This is the QBB case described above
993  if (!FBB)
994  FBB = std::next(MachineFunction::iterator(PBB));
995  }
996 
997  // Failing case: the only way IBB can be reached from PBB is via
998  // exception handling. Happens for landing pads. Would be nice to have
999  // a bit in the edge so we didn't have to do all this.
1000  if (IBB->isLandingPad()) {
1001  MachineFunction::iterator IP = PBB; IP++;
1002  MachineBasicBlock *PredNextBB = nullptr;
1003  if (IP != MF.end())
1004  PredNextBB = IP;
1005  if (!TBB) {
1006  if (IBB != PredNextBB) // fallthrough
1007  continue;
1008  } else if (FBB) {
1009  if (TBB != IBB && FBB != IBB) // cbr then ubr
1010  continue;
1011  } else if (Cond.empty()) {
1012  if (TBB != IBB) // ubr
1013  continue;
1014  } else {
1015  if (TBB != IBB && IBB != PredNextBB) // cbr
1016  continue;
1017  }
1018  }
1019 
1020  // Remove the unconditional branch at the end, if any.
1021  if (TBB && (Cond.empty() || FBB)) {
1022  DebugLoc dl; // FIXME: this is nowhere
1023  TII->RemoveBranch(*PBB);
1024  if (!Cond.empty())
1025  // reinsert conditional branch only, for now
1026  TII->InsertBranch(*PBB, (TBB == IBB) ? FBB : TBB, nullptr,
1027  NewCond, dl);
1028  }
1029 
1030  MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(PBB), *P));
1031  }
1032  }
1033 
1034  // If this is a large problem, avoid visiting the same basic blocks multiple
1035  // times.
1036  if (MergePotentials.size() == TailMergeThreshold)
1037  for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i)
1038  TriedMerging.insert(MergePotentials[i].getBlock());
1039 
1040  if (MergePotentials.size() >= 2)
1041  MadeChange |= TryTailMergeBlocks(IBB, PredBB);
1042 
1043  // Reinsert an unconditional branch if needed. The 1 below can occur as a
1044  // result of removing blocks in TryTailMergeBlocks.
1045  PredBB = std::prev(I); // this may have been changed in TryTailMergeBlocks
1046  if (MergePotentials.size() == 1 &&
1047  MergePotentials.begin()->getBlock() != PredBB)
1048  FixTail(MergePotentials.begin()->getBlock(), IBB, TII);
1049  }
1050 
1051  return MadeChange;
1052 }
1053 
1054 void BranchFolder::setCommonTailEdgeWeights(MachineBasicBlock &TailMBB) {
1055  SmallVector<BlockFrequency, 2> EdgeFreqLs(TailMBB.succ_size());
1056  BlockFrequency AccumulatedMBBFreq;
1057 
1058  // Aggregate edge frequency of successor edge j:
1059  // edgeFreq(j) = sum (freq(bb) * edgeProb(bb, j)),
1060  // where bb is a basic block that is in SameTails.
1061  for (const auto &Src : SameTails) {
1062  const MachineBasicBlock *SrcMBB = Src.getBlock();
1063  BlockFrequency BlockFreq = MBBFreqInfo.getBlockFreq(SrcMBB);
1064  AccumulatedMBBFreq += BlockFreq;
1065 
1066  // It is not necessary to recompute edge weights if TailBB has less than two
1067  // successors.
1068  if (TailMBB.succ_size() <= 1)
1069  continue;
1070 
1071  auto EdgeFreq = EdgeFreqLs.begin();
1072 
1073  for (auto SuccI = TailMBB.succ_begin(), SuccE = TailMBB.succ_end();
1074  SuccI != SuccE; ++SuccI, ++EdgeFreq)
1075  *EdgeFreq += BlockFreq * MBPI.getEdgeProbability(SrcMBB, *SuccI);
1076  }
1077 
1078  MBBFreqInfo.setBlockFreq(&TailMBB, AccumulatedMBBFreq);
1079 
1080  if (TailMBB.succ_size() <= 1)
1081  return;
1082 
1083  auto MaxEdgeFreq = *std::max_element(EdgeFreqLs.begin(), EdgeFreqLs.end());
1084  uint64_t Scale = MaxEdgeFreq.getFrequency() / UINT32_MAX + 1;
1085  auto EdgeFreq = EdgeFreqLs.begin();
1086 
1087  for (auto SuccI = TailMBB.succ_begin(), SuccE = TailMBB.succ_end();
1088  SuccI != SuccE; ++SuccI, ++EdgeFreq)
1089  TailMBB.setSuccWeight(SuccI, EdgeFreq->getFrequency() / Scale);
1090 }
1091 
1092 //===----------------------------------------------------------------------===//
1093 // Branch Optimization
1094 //===----------------------------------------------------------------------===//
1095 
1096 bool BranchFolder::OptimizeBranches(MachineFunction &MF) {
1097  bool MadeChange = false;
1098 
1099  // Make sure blocks are numbered in order
1100  MF.RenumberBlocks();
1101 
1102  for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end();
1103  I != E; ) {
1104  MachineBasicBlock *MBB = I++;
1105  MadeChange |= OptimizeBlock(MBB);
1106 
1107  // If it is dead, remove it.
1108  if (MBB->pred_empty()) {
1109  RemoveDeadBlock(MBB);
1110  MadeChange = true;
1111  ++NumDeadBlocks;
1112  }
1113  }
1114  return MadeChange;
1115 }
1116 
1117 // Blocks should be considered empty if they contain only debug info;
1118 // else the debug info would affect codegen.
1119 static bool IsEmptyBlock(MachineBasicBlock *MBB) {
1120  return MBB->getFirstNonDebugInstr() == MBB->end();
1121 }
1122 
1123 // Blocks with only debug info and branches should be considered the same
1124 // as blocks with only branches.
1127  assert(I != MBB->end() && "empty block!");
1128  return I->isBranch();
1129 }
1130 
1131 /// IsBetterFallthrough - Return true if it would be clearly better to
1132 /// fall-through to MBB1 than to fall through into MBB2. This has to return
1133 /// a strict ordering, returning true for both (MBB1,MBB2) and (MBB2,MBB1) will
1134 /// result in infinite loops.
1136  MachineBasicBlock *MBB2) {
1137  // Right now, we use a simple heuristic. If MBB2 ends with a call, and
1138  // MBB1 doesn't, we prefer to fall through into MBB1. This allows us to
1139  // optimize branches that branch to either a return block or an assert block
1140  // into a fallthrough to the return.
1143  if (MBB1I == MBB1->end() || MBB2I == MBB2->end())
1144  return false;
1145 
1146  // If there is a clear successor ordering we make sure that one block
1147  // will fall through to the next
1148  if (MBB1->isSuccessor(MBB2)) return true;
1149  if (MBB2->isSuccessor(MBB1)) return false;
1150 
1151  return MBB2I->isCall() && !MBB1I->isCall();
1152 }
1153 
1154 /// getBranchDebugLoc - Find and return, if any, the DebugLoc of the branch
1155 /// instructions on the block.
1158  if (I != MBB.end() && I->isBranch())
1159  return I->getDebugLoc();
1160  return DebugLoc();
1161 }
1162 
1163 /// OptimizeBlock - Analyze and optimize control flow related to the specified
1164 /// block. This is never called on the entry block.
1165 bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
1166  bool MadeChange = false;
1167  MachineFunction &MF = *MBB->getParent();
1168 ReoptimizeBlock:
1169 
1170  MachineFunction::iterator FallThrough = MBB;
1171  ++FallThrough;
1172 
1173  // If this block is empty, make everyone use its fall-through, not the block
1174  // explicitly. Landing pads should not do this since the landing-pad table
1175  // points to this block. Blocks with their addresses taken shouldn't be
1176  // optimized away.
1177  if (IsEmptyBlock(MBB) && !MBB->isLandingPad() && !MBB->hasAddressTaken()) {
1178  // Dead block? Leave for cleanup later.
1179  if (MBB->pred_empty()) return MadeChange;
1180 
1181  if (FallThrough == MF.end()) {
1182  // TODO: Simplify preds to not branch here if possible!
1183  } else if (FallThrough->isLandingPad()) {
1184  // Don't rewrite to a landing pad fallthough. That could lead to the case
1185  // where a BB jumps to more than one landing pad.
1186  // TODO: Is it ever worth rewriting predecessors which don't already
1187  // jump to a landing pad, and so can safely jump to the fallthrough?
1188  } else {
1189  // Rewrite all predecessors of the old block to go to the fallthrough
1190  // instead.
1191  while (!MBB->pred_empty()) {
1192  MachineBasicBlock *Pred = *(MBB->pred_end()-1);
1193  Pred->ReplaceUsesOfBlockWith(MBB, FallThrough);
1194  }
1195  // If MBB was the target of a jump table, update jump tables to go to the
1196  // fallthrough instead.
1197  if (MachineJumpTableInfo *MJTI = MF.getJumpTableInfo())
1198  MJTI->ReplaceMBBInJumpTables(MBB, FallThrough);
1199  MadeChange = true;
1200  }
1201  return MadeChange;
1202  }
1203 
1204  // Check to see if we can simplify the terminator of the block before this
1205  // one.
1206  MachineBasicBlock &PrevBB = *std::prev(MachineFunction::iterator(MBB));
1207 
1208  MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr;
1210  bool PriorUnAnalyzable =
1211  TII->AnalyzeBranch(PrevBB, PriorTBB, PriorFBB, PriorCond, true);
1212  if (!PriorUnAnalyzable) {
1213  // If the CFG for the prior block has extra edges, remove them.
1214  MadeChange |= PrevBB.CorrectExtraCFGEdges(PriorTBB, PriorFBB,
1215  !PriorCond.empty());
1216 
1217  // If the previous branch is conditional and both conditions go to the same
1218  // destination, remove the branch, replacing it with an unconditional one or
1219  // a fall-through.
1220  if (PriorTBB && PriorTBB == PriorFBB) {
1221  DebugLoc dl = getBranchDebugLoc(PrevBB);
1222  TII->RemoveBranch(PrevBB);
1223  PriorCond.clear();
1224  if (PriorTBB != MBB)
1225  TII->InsertBranch(PrevBB, PriorTBB, nullptr, PriorCond, dl);
1226  MadeChange = true;
1227  ++NumBranchOpts;
1228  goto ReoptimizeBlock;
1229  }
1230 
1231  // If the previous block unconditionally falls through to this block and
1232  // this block has no other predecessors, move the contents of this block
1233  // into the prior block. This doesn't usually happen when SimplifyCFG
1234  // has been used, but it can happen if tail merging splits a fall-through
1235  // predecessor of a block.
1236  // This has to check PrevBB->succ_size() because EH edges are ignored by
1237  // AnalyzeBranch.
1238  if (PriorCond.empty() && !PriorTBB && MBB->pred_size() == 1 &&
1239  PrevBB.succ_size() == 1 &&
1240  !MBB->hasAddressTaken() && !MBB->isLandingPad()) {
1241  DEBUG(dbgs() << "\nMerging into block: " << PrevBB
1242  << "From MBB: " << *MBB);
1243  // Remove redundant DBG_VALUEs first.
1244  if (PrevBB.begin() != PrevBB.end()) {
1245  MachineBasicBlock::iterator PrevBBIter = PrevBB.end();
1246  --PrevBBIter;
1247  MachineBasicBlock::iterator MBBIter = MBB->begin();
1248  // Check if DBG_VALUE at the end of PrevBB is identical to the
1249  // DBG_VALUE at the beginning of MBB.
1250  while (PrevBBIter != PrevBB.begin() && MBBIter != MBB->end()
1251  && PrevBBIter->isDebugValue() && MBBIter->isDebugValue()) {
1252  if (!MBBIter->isIdenticalTo(PrevBBIter))
1253  break;
1254  MachineInstr *DuplicateDbg = MBBIter;
1255  ++MBBIter; -- PrevBBIter;
1256  DuplicateDbg->eraseFromParent();
1257  }
1258  }
1259  PrevBB.splice(PrevBB.end(), MBB, MBB->begin(), MBB->end());
1260  PrevBB.removeSuccessor(PrevBB.succ_begin());
1261  assert(PrevBB.succ_empty());
1262  PrevBB.transferSuccessors(MBB);
1263  MadeChange = true;
1264  return MadeChange;
1265  }
1266 
1267  // If the previous branch *only* branches to *this* block (conditional or
1268  // not) remove the branch.
1269  if (PriorTBB == MBB && !PriorFBB) {
1270  TII->RemoveBranch(PrevBB);
1271  MadeChange = true;
1272  ++NumBranchOpts;
1273  goto ReoptimizeBlock;
1274  }
1275 
1276  // If the prior block branches somewhere else on the condition and here if
1277  // the condition is false, remove the uncond second branch.
1278  if (PriorFBB == MBB) {
1279  DebugLoc dl = getBranchDebugLoc(PrevBB);
1280  TII->RemoveBranch(PrevBB);
1281  TII->InsertBranch(PrevBB, PriorTBB, nullptr, PriorCond, dl);
1282  MadeChange = true;
1283  ++NumBranchOpts;
1284  goto ReoptimizeBlock;
1285  }
1286 
1287  // If the prior block branches here on true and somewhere else on false, and
1288  // if the branch condition is reversible, reverse the branch to create a
1289  // fall-through.
1290  if (PriorTBB == MBB) {
1291  SmallVector<MachineOperand, 4> NewPriorCond(PriorCond);
1292  if (!TII->ReverseBranchCondition(NewPriorCond)) {
1293  DebugLoc dl = getBranchDebugLoc(PrevBB);
1294  TII->RemoveBranch(PrevBB);
1295  TII->InsertBranch(PrevBB, PriorFBB, nullptr, NewPriorCond, dl);
1296  MadeChange = true;
1297  ++NumBranchOpts;
1298  goto ReoptimizeBlock;
1299  }
1300  }
1301 
1302  // If this block has no successors (e.g. it is a return block or ends with
1303  // a call to a no-return function like abort or __cxa_throw) and if the pred
1304  // falls through into this block, and if it would otherwise fall through
1305  // into the block after this, move this block to the end of the function.
1306  //
1307  // We consider it more likely that execution will stay in the function (e.g.
1308  // due to loops) than it is to exit it. This asserts in loops etc, moving
1309  // the assert condition out of the loop body.
1310  if (MBB->succ_empty() && !PriorCond.empty() && !PriorFBB &&
1311  MachineFunction::iterator(PriorTBB) == FallThrough &&
1312  !MBB->canFallThrough()) {
1313  bool DoTransform = true;
1314 
1315  // We have to be careful that the succs of PredBB aren't both no-successor
1316  // blocks. If neither have successors and if PredBB is the second from
1317  // last block in the function, we'd just keep swapping the two blocks for
1318  // last. Only do the swap if one is clearly better to fall through than
1319  // the other.
1320  if (FallThrough == --MF.end() &&
1321  !IsBetterFallthrough(PriorTBB, MBB))
1322  DoTransform = false;
1323 
1324  if (DoTransform) {
1325  // Reverse the branch so we will fall through on the previous true cond.
1326  SmallVector<MachineOperand, 4> NewPriorCond(PriorCond);
1327  if (!TII->ReverseBranchCondition(NewPriorCond)) {
1328  DEBUG(dbgs() << "\nMoving MBB: " << *MBB
1329  << "To make fallthrough to: " << *PriorTBB << "\n");
1330 
1331  DebugLoc dl = getBranchDebugLoc(PrevBB);
1332  TII->RemoveBranch(PrevBB);
1333  TII->InsertBranch(PrevBB, MBB, nullptr, NewPriorCond, dl);
1334 
1335  // Move this block to the end of the function.
1336  MBB->moveAfter(--MF.end());
1337  MadeChange = true;
1338  ++NumBranchOpts;
1339  return MadeChange;
1340  }
1341  }
1342  }
1343  }
1344 
1345  // Analyze the branch in the current block.
1346  MachineBasicBlock *CurTBB = nullptr, *CurFBB = nullptr;
1348  bool CurUnAnalyzable= TII->AnalyzeBranch(*MBB, CurTBB, CurFBB, CurCond, true);
1349  if (!CurUnAnalyzable) {
1350  // If the CFG for the prior block has extra edges, remove them.
1351  MadeChange |= MBB->CorrectExtraCFGEdges(CurTBB, CurFBB, !CurCond.empty());
1352 
1353  // If this is a two-way branch, and the FBB branches to this block, reverse
1354  // the condition so the single-basic-block loop is faster. Instead of:
1355  // Loop: xxx; jcc Out; jmp Loop
1356  // we want:
1357  // Loop: xxx; jncc Loop; jmp Out
1358  if (CurTBB && CurFBB && CurFBB == MBB && CurTBB != MBB) {
1359  SmallVector<MachineOperand, 4> NewCond(CurCond);
1360  if (!TII->ReverseBranchCondition(NewCond)) {
1361  DebugLoc dl = getBranchDebugLoc(*MBB);
1362  TII->RemoveBranch(*MBB);
1363  TII->InsertBranch(*MBB, CurFBB, CurTBB, NewCond, dl);
1364  MadeChange = true;
1365  ++NumBranchOpts;
1366  goto ReoptimizeBlock;
1367  }
1368  }
1369 
1370  // If this branch is the only thing in its block, see if we can forward
1371  // other blocks across it.
1372  if (CurTBB && CurCond.empty() && !CurFBB &&
1373  IsBranchOnlyBlock(MBB) && CurTBB != MBB &&
1374  !MBB->hasAddressTaken()) {
1375  DebugLoc dl = getBranchDebugLoc(*MBB);
1376  // This block may contain just an unconditional branch. Because there can
1377  // be 'non-branch terminators' in the block, try removing the branch and
1378  // then seeing if the block is empty.
1379  TII->RemoveBranch(*MBB);
1380  // If the only things remaining in the block are debug info, remove these
1381  // as well, so this will behave the same as an empty block in non-debug
1382  // mode.
1383  if (IsEmptyBlock(MBB)) {
1384  // Make the block empty, losing the debug info (we could probably
1385  // improve this in some cases.)
1386  MBB->erase(MBB->begin(), MBB->end());
1387  }
1388  // If this block is just an unconditional branch to CurTBB, we can
1389  // usually completely eliminate the block. The only case we cannot
1390  // completely eliminate the block is when the block before this one
1391  // falls through into MBB and we can't understand the prior block's branch
1392  // condition.
1393  if (MBB->empty()) {
1394  bool PredHasNoFallThrough = !PrevBB.canFallThrough();
1395  if (PredHasNoFallThrough || !PriorUnAnalyzable ||
1396  !PrevBB.isSuccessor(MBB)) {
1397  // If the prior block falls through into us, turn it into an
1398  // explicit branch to us to make updates simpler.
1399  if (!PredHasNoFallThrough && PrevBB.isSuccessor(MBB) &&
1400  PriorTBB != MBB && PriorFBB != MBB) {
1401  if (!PriorTBB) {
1402  assert(PriorCond.empty() && !PriorFBB &&
1403  "Bad branch analysis");
1404  PriorTBB = MBB;
1405  } else {
1406  assert(!PriorFBB && "Machine CFG out of date!");
1407  PriorFBB = MBB;
1408  }
1409  DebugLoc pdl = getBranchDebugLoc(PrevBB);
1410  TII->RemoveBranch(PrevBB);
1411  TII->InsertBranch(PrevBB, PriorTBB, PriorFBB, PriorCond, pdl);
1412  }
1413 
1414  // Iterate through all the predecessors, revectoring each in-turn.
1415  size_t PI = 0;
1416  bool DidChange = false;
1417  bool HasBranchToSelf = false;
1418  while(PI != MBB->pred_size()) {
1419  MachineBasicBlock *PMBB = *(MBB->pred_begin() + PI);
1420  if (PMBB == MBB) {
1421  // If this block has an uncond branch to itself, leave it.
1422  ++PI;
1423  HasBranchToSelf = true;
1424  } else {
1425  DidChange = true;
1426  PMBB->ReplaceUsesOfBlockWith(MBB, CurTBB);
1427  // If this change resulted in PMBB ending in a conditional
1428  // branch where both conditions go to the same destination,
1429  // change this to an unconditional branch (and fix the CFG).
1430  MachineBasicBlock *NewCurTBB = nullptr, *NewCurFBB = nullptr;
1431  SmallVector<MachineOperand, 4> NewCurCond;
1432  bool NewCurUnAnalyzable = TII->AnalyzeBranch(*PMBB, NewCurTBB,
1433  NewCurFBB, NewCurCond, true);
1434  if (!NewCurUnAnalyzable && NewCurTBB && NewCurTBB == NewCurFBB) {
1435  DebugLoc pdl = getBranchDebugLoc(*PMBB);
1436  TII->RemoveBranch(*PMBB);
1437  NewCurCond.clear();
1438  TII->InsertBranch(*PMBB, NewCurTBB, nullptr, NewCurCond, pdl);
1439  MadeChange = true;
1440  ++NumBranchOpts;
1441  PMBB->CorrectExtraCFGEdges(NewCurTBB, nullptr, false);
1442  }
1443  }
1444  }
1445 
1446  // Change any jumptables to go to the new MBB.
1447  if (MachineJumpTableInfo *MJTI = MF.getJumpTableInfo())
1448  MJTI->ReplaceMBBInJumpTables(MBB, CurTBB);
1449  if (DidChange) {
1450  ++NumBranchOpts;
1451  MadeChange = true;
1452  if (!HasBranchToSelf) return MadeChange;
1453  }
1454  }
1455  }
1456 
1457  // Add the branch back if the block is more than just an uncond branch.
1458  TII->InsertBranch(*MBB, CurTBB, nullptr, CurCond, dl);
1459  }
1460  }
1461 
1462  // If the prior block doesn't fall through into this block, and if this
1463  // block doesn't fall through into some other block, see if we can find a
1464  // place to move this block where a fall-through will happen.
1465  if (!PrevBB.canFallThrough()) {
1466 
1467  // Now we know that there was no fall-through into this block, check to
1468  // see if it has a fall-through into its successor.
1469  bool CurFallsThru = MBB->canFallThrough();
1470 
1471  if (!MBB->isLandingPad()) {
1472  // Check all the predecessors of this block. If one of them has no fall
1473  // throughs, move this block right after it.
1475  E = MBB->pred_end(); PI != E; ++PI) {
1476  // Analyze the branch at the end of the pred.
1477  MachineBasicBlock *PredBB = *PI;
1478  MachineFunction::iterator PredFallthrough = PredBB; ++PredFallthrough;
1479  MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
1481  if (PredBB != MBB && !PredBB->canFallThrough() &&
1482  !TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)
1483  && (!CurFallsThru || !CurTBB || !CurFBB)
1484  && (!CurFallsThru || MBB->getNumber() >= PredBB->getNumber())) {
1485  // If the current block doesn't fall through, just move it.
1486  // If the current block can fall through and does not end with a
1487  // conditional branch, we need to append an unconditional jump to
1488  // the (current) next block. To avoid a possible compile-time
1489  // infinite loop, move blocks only backward in this case.
1490  // Also, if there are already 2 branches here, we cannot add a third;
1491  // this means we have the case
1492  // Bcc next
1493  // B elsewhere
1494  // next:
1495  if (CurFallsThru) {
1496  MachineBasicBlock *NextBB =
1497  std::next(MachineFunction::iterator(MBB));
1498  CurCond.clear();
1499  TII->InsertBranch(*MBB, NextBB, nullptr, CurCond, DebugLoc());
1500  }
1501  MBB->moveAfter(PredBB);
1502  MadeChange = true;
1503  goto ReoptimizeBlock;
1504  }
1505  }
1506  }
1507 
1508  if (!CurFallsThru) {
1509  // Check all successors to see if we can move this block before it.
1511  E = MBB->succ_end(); SI != E; ++SI) {
1512  // Analyze the branch at the end of the block before the succ.
1513  MachineBasicBlock *SuccBB = *SI;
1514  MachineFunction::iterator SuccPrev = SuccBB; --SuccPrev;
1515 
1516  // If this block doesn't already fall-through to that successor, and if
1517  // the succ doesn't already have a block that can fall through into it,
1518  // and if the successor isn't an EH destination, we can arrange for the
1519  // fallthrough to happen.
1520  if (SuccBB != MBB && &*SuccPrev != MBB &&
1521  !SuccPrev->canFallThrough() && !CurUnAnalyzable &&
1522  !SuccBB->isLandingPad()) {
1523  MBB->moveBefore(SuccBB);
1524  MadeChange = true;
1525  goto ReoptimizeBlock;
1526  }
1527  }
1528 
1529  // Okay, there is no really great place to put this block. If, however,
1530  // the block before this one would be a fall-through if this block were
1531  // removed, move this block to the end of the function.
1532  MachineBasicBlock *PrevTBB = nullptr, *PrevFBB = nullptr;
1534  if (FallThrough != MF.end() &&
1535  !TII->AnalyzeBranch(PrevBB, PrevTBB, PrevFBB, PrevCond, true) &&
1536  PrevBB.isSuccessor(FallThrough)) {
1537  MBB->moveAfter(--MF.end());
1538  MadeChange = true;
1539  return MadeChange;
1540  }
1541  }
1542  }
1543 
1544  return MadeChange;
1545 }
1546 
1547 //===----------------------------------------------------------------------===//
1548 // Hoist Common Code
1549 //===----------------------------------------------------------------------===//
1550 
1551 /// HoistCommonCode - Hoist common instruction sequences at the start of basic
1552 /// blocks to their common predecessor.
1553 bool BranchFolder::HoistCommonCode(MachineFunction &MF) {
1554  bool MadeChange = false;
1555  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ) {
1556  MachineBasicBlock *MBB = I++;
1557  MadeChange |= HoistCommonCodeInSuccs(MBB);
1558  }
1559 
1560  return MadeChange;
1561 }
1562 
1563 /// findFalseBlock - BB has a fallthrough. Find its 'false' successor given
1564 /// its 'true' successor.
1566  MachineBasicBlock *TrueBB) {
1568  E = BB->succ_end(); SI != E; ++SI) {
1569  MachineBasicBlock *SuccBB = *SI;
1570  if (SuccBB != TrueBB)
1571  return SuccBB;
1572  }
1573  return nullptr;
1574 }
1575 
1576 /// findHoistingInsertPosAndDeps - Find the location to move common instructions
1577 /// in successors to. The location is usually just before the terminator,
1578 /// however if the terminator is a conditional branch and its previous
1579 /// instruction is the flag setting instruction, the previous instruction is
1580 /// the preferred location. This function also gathers uses and defs of the
1581 /// instructions from the insertion point to the end of the block. The data is
1582 /// used by HoistCommonCodeInSuccs to ensure safety.
1583 static
1585  const TargetInstrInfo *TII,
1586  const TargetRegisterInfo *TRI,
1587  SmallSet<unsigned,4> &Uses,
1588  SmallSet<unsigned,4> &Defs) {
1590  if (!TII->isUnpredicatedTerminator(Loc))
1591  return MBB->end();
1592 
1593  for (unsigned i = 0, e = Loc->getNumOperands(); i != e; ++i) {
1594  const MachineOperand &MO = Loc->getOperand(i);
1595  if (!MO.isReg())
1596  continue;
1597  unsigned Reg = MO.getReg();
1598  if (!Reg)
1599  continue;
1600  if (MO.isUse()) {
1601  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1602  Uses.insert(*AI);
1603  } else {
1604  if (!MO.isDead())
1605  // Don't try to hoist code in the rare case the terminator defines a
1606  // register that is later used.
1607  return MBB->end();
1608 
1609  // If the terminator defines a register, make sure we don't hoist
1610  // the instruction whose def might be clobbered by the terminator.
1611  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1612  Defs.insert(*AI);
1613  }
1614  }
1615 
1616  if (Uses.empty())
1617  return Loc;
1618  if (Loc == MBB->begin())
1619  return MBB->end();
1620 
1621  // The terminator is probably a conditional branch, try not to separate the
1622  // branch from condition setting instruction.
1623  MachineBasicBlock::iterator PI = Loc;
1624  --PI;
1625  while (PI != MBB->begin() && PI->isDebugValue())
1626  --PI;
1627 
1628  bool IsDef = false;
1629  for (unsigned i = 0, e = PI->getNumOperands(); !IsDef && i != e; ++i) {
1630  const MachineOperand &MO = PI->getOperand(i);
1631  // If PI has a regmask operand, it is probably a call. Separate away.
1632  if (MO.isRegMask())
1633  return Loc;
1634  if (!MO.isReg() || MO.isUse())
1635  continue;
1636  unsigned Reg = MO.getReg();
1637  if (!Reg)
1638  continue;
1639  if (Uses.count(Reg))
1640  IsDef = true;
1641  }
1642  if (!IsDef)
1643  // The condition setting instruction is not just before the conditional
1644  // branch.
1645  return Loc;
1646 
1647  // Be conservative, don't insert instruction above something that may have
1648  // side-effects. And since it's potentially bad to separate flag setting
1649  // instruction from the conditional branch, just abort the optimization
1650  // completely.
1651  // Also avoid moving code above predicated instruction since it's hard to
1652  // reason about register liveness with predicated instruction.
1653  bool DontMoveAcrossStore = true;
1654  if (!PI->isSafeToMove(nullptr, DontMoveAcrossStore) || TII->isPredicated(PI))
1655  return MBB->end();
1656 
1657 
1658  // Find out what registers are live. Note this routine is ignoring other live
1659  // registers which are only used by instructions in successor blocks.
1660  for (unsigned i = 0, e = PI->getNumOperands(); i != e; ++i) {
1661  const MachineOperand &MO = PI->getOperand(i);
1662  if (!MO.isReg())
1663  continue;
1664  unsigned Reg = MO.getReg();
1665  if (!Reg)
1666  continue;
1667  if (MO.isUse()) {
1668  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1669  Uses.insert(*AI);
1670  } else {
1671  if (Uses.erase(Reg)) {
1672  for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
1673  Uses.erase(*SubRegs); // Use sub-registers to be conservative
1674  }
1675  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1676  Defs.insert(*AI);
1677  }
1678  }
1679 
1680  return PI;
1681 }
1682 
1683 /// HoistCommonCodeInSuccs - If the successors of MBB has common instruction
1684 /// sequence at the start of the function, move the instructions before MBB
1685 /// terminator if it's legal.
1686 bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
1687  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
1689  if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, true) || !TBB || Cond.empty())
1690  return false;
1691 
1692  if (!FBB) FBB = findFalseBlock(MBB, TBB);
1693  if (!FBB)
1694  // Malformed bcc? True and false blocks are the same?
1695  return false;
1696 
1697  // Restrict the optimization to cases where MBB is the only predecessor,
1698  // it is an obvious win.
1699  if (TBB->pred_size() > 1 || FBB->pred_size() > 1)
1700  return false;
1701 
1702  // Find a suitable position to hoist the common instructions to. Also figure
1703  // out which registers are used or defined by instructions from the insertion
1704  // point to the end of the block.
1705  SmallSet<unsigned, 4> Uses, Defs;
1707  findHoistingInsertPosAndDeps(MBB, TII, TRI, Uses, Defs);
1708  if (Loc == MBB->end())
1709  return false;
1710 
1711  bool HasDups = false;
1712  SmallVector<unsigned, 4> LocalDefs;
1713  SmallSet<unsigned, 4> LocalDefsSet;
1714  MachineBasicBlock::iterator TIB = TBB->begin();
1715  MachineBasicBlock::iterator FIB = FBB->begin();
1716  MachineBasicBlock::iterator TIE = TBB->end();
1717  MachineBasicBlock::iterator FIE = FBB->end();
1718  while (TIB != TIE && FIB != FIE) {
1719  // Skip dbg_value instructions. These do not count.
1720  if (TIB->isDebugValue()) {
1721  while (TIB != TIE && TIB->isDebugValue())
1722  ++TIB;
1723  if (TIB == TIE)
1724  break;
1725  }
1726  if (FIB->isDebugValue()) {
1727  while (FIB != FIE && FIB->isDebugValue())
1728  ++FIB;
1729  if (FIB == FIE)
1730  break;
1731  }
1732  if (!TIB->isIdenticalTo(FIB, MachineInstr::CheckKillDead))
1733  break;
1734 
1735  if (TII->isPredicated(TIB))
1736  // Hard to reason about register liveness with predicated instruction.
1737  break;
1738 
1739  bool IsSafe = true;
1740  for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) {
1741  MachineOperand &MO = TIB->getOperand(i);
1742  // Don't attempt to hoist instructions with register masks.
1743  if (MO.isRegMask()) {
1744  IsSafe = false;
1745  break;
1746  }
1747  if (!MO.isReg())
1748  continue;
1749  unsigned Reg = MO.getReg();
1750  if (!Reg)
1751  continue;
1752  if (MO.isDef()) {
1753  if (Uses.count(Reg)) {
1754  // Avoid clobbering a register that's used by the instruction at
1755  // the point of insertion.
1756  IsSafe = false;
1757  break;
1758  }
1759 
1760  if (Defs.count(Reg) && !MO.isDead()) {
1761  // Don't hoist the instruction if the def would be clobber by the
1762  // instruction at the point insertion. FIXME: This is overly
1763  // conservative. It should be possible to hoist the instructions
1764  // in BB2 in the following example:
1765  // BB1:
1766  // r1, eflag = op1 r2, r3
1767  // brcc eflag
1768  //
1769  // BB2:
1770  // r1 = op2, ...
1771  // = op3, r1<kill>
1772  IsSafe = false;
1773  break;
1774  }
1775  } else if (!LocalDefsSet.count(Reg)) {
1776  if (Defs.count(Reg)) {
1777  // Use is defined by the instruction at the point of insertion.
1778  IsSafe = false;
1779  break;
1780  }
1781 
1782  if (MO.isKill() && Uses.count(Reg))
1783  // Kills a register that's read by the instruction at the point of
1784  // insertion. Remove the kill marker.
1785  MO.setIsKill(false);
1786  }
1787  }
1788  if (!IsSafe)
1789  break;
1790 
1791  bool DontMoveAcrossStore = true;
1792  if (!TIB->isSafeToMove(nullptr, DontMoveAcrossStore))
1793  break;
1794 
1795  // Remove kills from LocalDefsSet, these registers had short live ranges.
1796  for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) {
1797  MachineOperand &MO = TIB->getOperand(i);
1798  if (!MO.isReg() || !MO.isUse() || !MO.isKill())
1799  continue;
1800  unsigned Reg = MO.getReg();
1801  if (!Reg || !LocalDefsSet.count(Reg))
1802  continue;
1803  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1804  LocalDefsSet.erase(*AI);
1805  }
1806 
1807  // Track local defs so we can update liveins.
1808  for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) {
1809  MachineOperand &MO = TIB->getOperand(i);
1810  if (!MO.isReg() || !MO.isDef() || MO.isDead())
1811  continue;
1812  unsigned Reg = MO.getReg();
1813  if (!Reg)
1814  continue;
1815  LocalDefs.push_back(Reg);
1816  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1817  LocalDefsSet.insert(*AI);
1818  }
1819 
1820  HasDups = true;
1821  ++TIB;
1822  ++FIB;
1823  }
1824 
1825  if (!HasDups)
1826  return false;
1827 
1828  MBB->splice(Loc, TBB, TBB->begin(), TIB);
1829  FBB->erase(FBB->begin(), FIB);
1830 
1831  // Update livein's.
1832  for (unsigned i = 0, e = LocalDefs.size(); i != e; ++i) {
1833  unsigned Def = LocalDefs[i];
1834  if (LocalDefsSet.count(Def)) {
1835  TBB->addLiveIn(Def);
1836  FBB->addLiveIn(Def);
1837  }
1838  }
1839 
1840  ++NumHoist;
1841  return true;
1842 }
unsigned succ_size() const
static unsigned EstimateRuntime(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
EstimateRuntime - Make a rough estimate for how long it will take to run the specified code...
void push_back(const T &Elt)
Definition: SmallVector.h:222
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
static cl::opt< unsigned > TailMergeThreshold("tail-merge-threshold", cl::desc("Max number of predecessors to consider tail merging"), cl::init(150), cl::Hidden)
static void removeMMOsFromMemoryOperations(MachineBasicBlock::iterator MBBIStartPos, MachineBasicBlock &MBBCommon)
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them...
MachineBasicBlock * getMBB() const
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
iterator getFirstNonDebugInstr()
getFirstNonDebugInstr - returns an iterator to the first non-debug instruction in the basic block...
iterator getFirstTerminator()
getFirstTerminator - returns an iterator to the first terminator instruction of this basic block...
bool isDead() const
Address of indexed Jump Table for switch.
virtual bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
void RemoveJumpTable(unsigned Idx)
RemoveJumpTable - Mark the specific index as being dead.
void addLiveIn(unsigned Reg)
Adds the specified register as a live in.
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
const_iterator begin(StringRef path)
Get begin iterator over path.
Definition: Path.cpp:232
MachineBasicBlock reference.
void moveAfter(MachineBasicBlock *NewBefore)
A debug info location.
Definition: DebugLoc.h:34
F(f)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
virtual unsigned RemoveBranch(MachineBasicBlock &MBB) const
Remove the branching code at the end of the specific MBB.
static DebugLoc getBranchDebugLoc(MachineBasicBlock &MBB)
getBranchDebugLoc - Find and return, if any, the DebugLoc of the branch instructions on the block...
#define op(i)
bool erase(const T &V)
Definition: SmallSet.h:96
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallSet.h:44
const std::vector< MachineJumpTableEntry > & getJumpTables() const
bool isJTI() const
isJTI - Tests if this is a MO_JumpTableIndex operand.
AnalysisUsage & addRequired()
void ReplaceUsesOfBlockWith(MachineBasicBlock *Old, MachineBasicBlock *New)
ReplaceUsesOfBlockWith - Given a machine basic block that branched to 'Old', change the code and CFG ...
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Name of external global symbol.
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
INITIALIZE_PASS(BranchFolderPass,"branch-folder","Control Flow Optimizer", false, false) bool BranchFolderPass
Target-Independent Code Generator Pass Configuration Options.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
void forward()
Move the internal MBB iterator and update register states.
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
bool isKill() const
void transferSuccessors(MachineBasicBlock *fromMBB)
transferSuccessors - Transfers all the successors from MBB to this machine basic block (i...
static cl::opt< cl::boolOrDefault > FlagEnableTailMerge("enable-tail-merge", cl::init(cl::BOU_UNSET), cl::Hidden)
static void FixTail(MachineBasicBlock *CurMBB, MachineBasicBlock *SuccBB, const TargetInstrInfo *TII)
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
bool canFallThrough()
canFallThrough - Return true if the block can implicitly transfer control to the block after it by fa...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
iterator getLastNonDebugInstr()
getLastNonDebugInstr - returns an iterator to the last non-debug instruction in the basic block...
void enterBasicBlock(MachineBasicBlock *mbb)
Start tracking liveness from the begin of the specific basic block.
virtual bool ReverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const
Reverses the branch condition of the specified condition list, returning false on success and true if...
std::vector< MachineBasicBlock * >::iterator pred_iterator
static MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, SmallSet< unsigned, 4 > &Uses, SmallSet< unsigned, 4 > &Defs)
findHoistingInsertPosAndDeps - Find the location to move common instructions in successors to...
int64_t getImm() const
iterator begin()
Definition: Function.h:457
reverse_iterator rend()
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
reverse_iterator rbegin()
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:267
TargetInstrInfo - Interface to description of machine instruction set.
mmo_iterator memoperands_end() const
Definition: MachineInstr.h:341
bundle_iterator< MachineInstr, instr_iterator > iterator
#define P(N)
Address of a global value.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition: STLExtras.h:287
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
bundle_iterator - MachineBasicBlock iterator that automatically skips over MIs that are inside bundle...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
bool getEnableTailMerge() const
const MachineBasicBlock * getLandingPadSuccessor() const
getLandingPadSuccessor - If this block has a successor that is a landing pad, return it...
int64_t getOffset() const
Return the offset from the symbol in this operand.
static bool IsBetterFallthrough(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2)
IsBetterFallthrough - Return true if it would be clearly better to fall-through to MBB1 than to fall ...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
void removeSuccessor(MachineBasicBlock *succ)
removeSuccessor - Remove successor from the successors list of this MachineBasicBlock.
void moveBefore(MachineBasicBlock *NewAfter)
moveBefore/moveAfter - move 'this' block before or after the specified block.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
MCSubRegIterator enumerates all sub-registers of Reg.
static unsigned CountTerminators(MachineBasicBlock *MBB, MachineBasicBlock::iterator &I)
CountTerminators - Count the number of terminators in the given block and set I to the position of th...
static cl::opt< unsigned > TailMergeSize("tail-merge-size", cl::desc("Min number of instructions to consider tail merging"), cl::init(3), cl::Hidden)
void setIsKill(bool Val=true)
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:53
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
static bool hasIdenticalMMOs(const MachineInstr *MI1, const MachineInstr *MI2)
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
static bool ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2, unsigned minCommonTailLength, unsigned &CommonTailLen, MachineBasicBlock::iterator &I1, MachineBasicBlock::iterator &I2, MachineBasicBlock *SuccBB, MachineBasicBlock *PredBB)
ProfitableToMerge - Check if two machine basic blocks have a common tail and decide if it would be pr...
bool isRegUsed(unsigned Reg, bool includeReserved=true) const
Return if a specific register is currently used.
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
virtual bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const
trackLivenessAfterRegAlloc - returns true if the live-ins should be tracked after register allocation...
virtual bool isUnpredicatedTerminator(const MachineInstr *MI) const
Returns true if the instruction is a terminator instruction that has not been predicated.
void invalidateLiveness()
invalidateLiveness - Indicates that register liveness is no longer being tracked accurately.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
virtual bool isLegalToSplitMBBAt(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const
Return true if it's legal to split the given basic block at the specified instruction (i...
Representation of each machine instruction.
Definition: MachineInstr.h:51
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
bundle_iterator< const MachineInstr, const_instr_iterator > const_iterator
bool hasAddressTaken() const
hasAddressTaken - Test whether this block is potentially the target of an indirect branch...
bool isLandingPad() const
isLandingPad - Returns true if the block is a landing pad.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:217
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
#define I(x, y, z)
Definition: MD5.cpp:54
static bool IsBranchOnlyBlock(MachineBasicBlock *MBB)
virtual unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, DebugLoc DL) const
Insert branch code into the end of the specified MachineBasicBlock.
Abstract Stack Frame Index.
void setSuccWeight(succ_iterator I, uint32_t weight)
Set successor weight of a given iterator.
BranchFolder(bool defaultEnableTailMerge, bool CommonHoist, const MachineBlockFrequencyInfo &MBFI, const MachineBranchProbabilityInfo &MBPI)
static MachineBasicBlock * findFalseBlock(MachineBasicBlock *BB, MachineBasicBlock *TrueBB)
findFalseBlock - BB has a fallthrough.
unsigned getReg() const
getReg - Returns the register number.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:332
std::reverse_iterator< iterator > reverse_iterator
static const Function * getParent(const Value *V)
BasicBlockListType::iterator iterator
char & BranchFolderPassID
BranchFolding - This pass performs machine code CFG based optimizations to delete branches to branche...
#define DEBUG(X)
Definition: Debug.h:92
static unsigned HashMachineInstr(const MachineInstr *MI)
HashMachineInstr - Compute a hash value for MI and its operands.
std::string Hash(const Unit &U)
Definition: FuzzerUtil.cpp:39
virtual bool isPredicated(const MachineInstr *MI) const
Returns true if the instruction is already predicated.
Address of indexed Constant in Constant Pool.
bool OptimizeFunction(MachineFunction &MF, const TargetInstrInfo *tii, const TargetRegisterInfo *tri, MachineModuleInfo *mmi)
OptimizeFunction - Perhaps branch folding, tail merging and other CFG optimizations on the given func...
static unsigned HashEndOfMBB(const MachineBasicBlock *MBB)
HashEndOfMBB - Hash the last instruction in the MBB.
virtual void ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, MachineBasicBlock *NewDest) const
Delete the instruction OldInst and everything after it, replacing it with an unconditional branch to ...
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
isLayoutSuccessor - Return true if the specified MBB will be emitted immediately after this block...
void addSuccessor(MachineBasicBlock *succ, uint32_t weight=0)
addSuccessor - Add succ as a successor of this MachineBasicBlock.
unsigned pred_size() const
bool CorrectExtraCFGEdges(MachineBasicBlock *DestA, MachineBasicBlock *DestB, bool isCond)
CorrectExtraCFGEdges - Various pieces of code can cause excess edges in the CFG to be inserted...
bool isBarrier(QueryType Type=AnyInBundle) const
Returns true if the specified instruction stops control flow from executing the instruction immediate...
Definition: MachineInstr.h:410
MachineModuleInfo - This class contains meta information specific to a module.
static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2, MachineBasicBlock::iterator &I1, MachineBasicBlock::iterator &I2)
ComputeCommonTailLength - Given two machine basic blocks, compute the number of instructions they act...
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:340