LLVM  3.7.0
TailDuplication.cpp
Go to the documentation of this file.
1 //===-- TailDuplication.cpp - Duplicate blocks into predecessors' tails ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass duplicates basic blocks ending in unconditional branches into
11 // the tails of their predecessors.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/CodeGen/Passes.h"
16 #include "llvm/ADT/DenseSet.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
27 #include "llvm/IR/Function.h"
29 #include "llvm/Support/Debug.h"
35 using namespace llvm;
36 
37 #define DEBUG_TYPE "tailduplication"
38 
39 STATISTIC(NumTails , "Number of tails duplicated");
40 STATISTIC(NumTailDups , "Number of tail duplicated blocks");
41 STATISTIC(NumInstrDups , "Additional instructions due to tail duplication");
42 STATISTIC(NumDeadBlocks, "Number of dead blocks removed");
43 STATISTIC(NumAddedPHIs , "Number of phis added");
44 
45 // Heuristic for tail duplication.
46 static cl::opt<unsigned>
47 TailDuplicateSize("tail-dup-size",
48  cl::desc("Maximum instructions to consider tail duplicating"),
49  cl::init(2), cl::Hidden);
50 
51 static cl::opt<bool>
52 TailDupVerify("tail-dup-verify",
53  cl::desc("Verify sanity of PHI instructions during taildup"),
54  cl::init(false), cl::Hidden);
55 
56 static cl::opt<unsigned>
57 TailDupLimit("tail-dup-limit", cl::init(~0U), cl::Hidden);
58 
59 typedef std::vector<std::pair<MachineBasicBlock*,unsigned> > AvailableValsTy;
60 
61 namespace {
62  /// TailDuplicatePass - Perform tail duplication.
63  class TailDuplicatePass : public MachineFunctionPass {
64  const TargetInstrInfo *TII;
65  const TargetRegisterInfo *TRI;
66  const MachineBranchProbabilityInfo *MBPI;
67  MachineModuleInfo *MMI;
69  std::unique_ptr<RegScavenger> RS;
70  bool PreRegAlloc;
71 
72  // SSAUpdateVRs - A list of virtual registers for which to update SSA form.
73  SmallVector<unsigned, 16> SSAUpdateVRs;
74 
75  // SSAUpdateVals - For each virtual register in SSAUpdateVals keep a list of
76  // source virtual registers.
78 
79  public:
80  static char ID;
81  explicit TailDuplicatePass() :
82  MachineFunctionPass(ID), PreRegAlloc(false) {}
83 
84  bool runOnMachineFunction(MachineFunction &MF) override;
85 
86  void getAnalysisUsage(AnalysisUsage &AU) const override;
87 
88  private:
89  void AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg,
90  MachineBasicBlock *BB);
91  void ProcessPHI(MachineInstr *MI, MachineBasicBlock *TailBB,
92  MachineBasicBlock *PredBB,
93  DenseMap<unsigned, unsigned> &LocalVRMap,
94  SmallVectorImpl<std::pair<unsigned,unsigned> > &Copies,
95  const DenseSet<unsigned> &UsedByPhi,
96  bool Remove);
97  void DuplicateInstruction(MachineInstr *MI,
98  MachineBasicBlock *TailBB,
99  MachineBasicBlock *PredBB,
100  MachineFunction &MF,
101  DenseMap<unsigned, unsigned> &LocalVRMap,
102  const DenseSet<unsigned> &UsedByPhi);
103  void UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead,
106  bool TailDuplicateBlocks(MachineFunction &MF);
107  bool shouldTailDuplicate(const MachineFunction &MF,
108  bool IsSimple, MachineBasicBlock &TailBB);
109  bool isSimpleBB(MachineBasicBlock *TailBB);
110  bool canCompletelyDuplicateBB(MachineBasicBlock &BB);
111  bool duplicateSimpleBB(MachineBasicBlock *TailBB,
113  const DenseSet<unsigned> &RegsUsedByPhi,
115  bool TailDuplicate(MachineBasicBlock *TailBB,
116  bool IsSimple,
117  MachineFunction &MF,
120  bool TailDuplicateAndUpdate(MachineBasicBlock *MBB,
121  bool IsSimple,
122  MachineFunction &MF);
123 
124  void RemoveDeadBlock(MachineBasicBlock *MBB);
125  };
126 
127  char TailDuplicatePass::ID = 0;
128 }
129 
131 
132 INITIALIZE_PASS(TailDuplicatePass, "tailduplication", "Tail Duplication",
133  false, false)
134 
135 bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) {
136  if (skipOptnoneFunction(*MF.getFunction()))
137  return false;
138 
139  TII = MF.getSubtarget().getInstrInfo();
140  TRI = MF.getSubtarget().getRegisterInfo();
141  MRI = &MF.getRegInfo();
142  MMI = getAnalysisIfAvailable<MachineModuleInfo>();
143  MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
144 
145  PreRegAlloc = MRI->isSSA();
146  RS.reset();
147  if (MRI->tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF))
148  RS.reset(new RegScavenger());
149 
150  bool MadeChange = false;
151  while (TailDuplicateBlocks(MF))
152  MadeChange = true;
153 
154  return MadeChange;
155 }
156 
157 void TailDuplicatePass::getAnalysisUsage(AnalysisUsage &AU) const {
160 }
161 
162 static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) {
163  for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) {
164  MachineBasicBlock *MBB = I;
166  MBB->pred_end());
168  while (MI != MBB->end()) {
169  if (!MI->isPHI())
170  break;
172  PE = Preds.end(); PI != PE; ++PI) {
173  MachineBasicBlock *PredBB = *PI;
174  bool Found = false;
175  for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
176  MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
177  if (PHIBB == PredBB) {
178  Found = true;
179  break;
180  }
181  }
182  if (!Found) {
183  dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
184  dbgs() << " missing input from predecessor BB#"
185  << PredBB->getNumber() << '\n';
186  llvm_unreachable(nullptr);
187  }
188  }
189 
190  for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
191  MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
192  if (CheckExtra && !Preds.count(PHIBB)) {
193  dbgs() << "Warning: malformed PHI in BB#" << MBB->getNumber()
194  << ": " << *MI;
195  dbgs() << " extra input from predecessor BB#"
196  << PHIBB->getNumber() << '\n';
197  llvm_unreachable(nullptr);
198  }
199  if (PHIBB->getNumber() < 0) {
200  dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
201  dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n';
202  llvm_unreachable(nullptr);
203  }
204  }
205  ++MI;
206  }
207  }
208 }
209 
210 /// TailDuplicateAndUpdate - Tail duplicate the block and cleanup.
211 bool
212 TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB,
213  bool IsSimple,
214  MachineFunction &MF) {
215  // Save the successors list.
217  MBB->succ_end());
218 
221  if (!TailDuplicate(MBB, IsSimple, MF, TDBBs, Copies))
222  return false;
223 
224  ++NumTails;
225 
227  MachineSSAUpdater SSAUpdate(MF, &NewPHIs);
228 
229  // TailBB's immediate successors are now successors of those predecessors
230  // which duplicated TailBB. Add the predecessors as sources to the PHI
231  // instructions.
232  bool isDead = MBB->pred_empty() && !MBB->hasAddressTaken();
233  if (PreRegAlloc)
234  UpdateSuccessorsPHIs(MBB, isDead, TDBBs, Succs);
235 
236  // If it is dead, remove it.
237  if (isDead) {
238  NumInstrDups -= MBB->size();
239  RemoveDeadBlock(MBB);
240  ++NumDeadBlocks;
241  }
242 
243  // Update SSA form.
244  if (!SSAUpdateVRs.empty()) {
245  for (unsigned i = 0, e = SSAUpdateVRs.size(); i != e; ++i) {
246  unsigned VReg = SSAUpdateVRs[i];
247  SSAUpdate.Initialize(VReg);
248 
249  // If the original definition is still around, add it as an available
250  // value.
251  MachineInstr *DefMI = MRI->getVRegDef(VReg);
252  MachineBasicBlock *DefBB = nullptr;
253  if (DefMI) {
254  DefBB = DefMI->getParent();
255  SSAUpdate.AddAvailableValue(DefBB, VReg);
256  }
257 
258  // Add the new vregs as available values.
260  SSAUpdateVals.find(VReg);
261  for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
262  MachineBasicBlock *SrcBB = LI->second[j].first;
263  unsigned SrcReg = LI->second[j].second;
264  SSAUpdate.AddAvailableValue(SrcBB, SrcReg);
265  }
266 
267  // Rewrite uses that are outside of the original def's block.
268  MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg);
269  while (UI != MRI->use_end()) {
270  MachineOperand &UseMO = *UI;
271  MachineInstr *UseMI = UseMO.getParent();
272  ++UI;
273  if (UseMI->isDebugValue()) {
274  // SSAUpdate can replace the use with an undef. That creates
275  // a debug instruction that is a kill.
276  // FIXME: Should it SSAUpdate job to delete debug instructions
277  // instead of replacing the use with undef?
278  UseMI->eraseFromParent();
279  continue;
280  }
281  if (UseMI->getParent() == DefBB && !UseMI->isPHI())
282  continue;
283  SSAUpdate.RewriteUse(UseMO);
284  }
285  }
286 
287  SSAUpdateVRs.clear();
288  SSAUpdateVals.clear();
289  }
290 
291  // Eliminate some of the copies inserted by tail duplication to maintain
292  // SSA form.
293  for (unsigned i = 0, e = Copies.size(); i != e; ++i) {
294  MachineInstr *Copy = Copies[i];
295  if (!Copy->isCopy())
296  continue;
297  unsigned Dst = Copy->getOperand(0).getReg();
298  unsigned Src = Copy->getOperand(1).getReg();
299  if (MRI->hasOneNonDBGUse(Src) &&
300  MRI->constrainRegClass(Src, MRI->getRegClass(Dst))) {
301  // Copy is the only use. Do trivial copy propagation here.
302  MRI->replaceRegWith(Dst, Src);
303  Copy->eraseFromParent();
304  }
305  }
306 
307  if (NewPHIs.size())
308  NumAddedPHIs += NewPHIs.size();
309 
310  return true;
311 }
312 
313 /// TailDuplicateBlocks - Look for small blocks that are unconditionally
314 /// branched to and do not fall through. Tail-duplicate their instructions
315 /// into their predecessors to eliminate (dynamic) branches.
316 bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) {
317  bool MadeChange = false;
318 
319  if (PreRegAlloc && TailDupVerify) {
320  DEBUG(dbgs() << "\n*** Before tail-duplicating\n");
321  VerifyPHIs(MF, true);
322  }
323 
324  for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ) {
325  MachineBasicBlock *MBB = I++;
326 
327  if (NumTails == TailDupLimit)
328  break;
329 
330  bool IsSimple = isSimpleBB(MBB);
331 
332  if (!shouldTailDuplicate(MF, IsSimple, *MBB))
333  continue;
334 
335  MadeChange |= TailDuplicateAndUpdate(MBB, IsSimple, MF);
336  }
337 
338  if (PreRegAlloc && TailDupVerify)
339  VerifyPHIs(MF, false);
340 
341  return MadeChange;
342 }
343 
344 static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB,
345  const MachineRegisterInfo *MRI) {
346  for (MachineInstr &UseMI : MRI->use_instructions(Reg)) {
347  if (UseMI.isDebugValue())
348  continue;
349  if (UseMI.getParent() != BB)
350  return true;
351  }
352  return false;
353 }
354 
356  for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2)
357  if (MI->getOperand(i+1).getMBB() == SrcBB)
358  return i;
359  return 0;
360 }
361 
362 
363 // Remember which registers are used by phis in this block. This is
364 // used to determine which registers are liveout while modifying the
365 // block (which is why we need to copy the information).
366 static void getRegsUsedByPHIs(const MachineBasicBlock &BB,
367  DenseSet<unsigned> *UsedByPhi) {
368  for (const auto &MI : BB) {
369  if (!MI.isPHI())
370  break;
371  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
372  unsigned SrcReg = MI.getOperand(i).getReg();
373  UsedByPhi->insert(SrcReg);
374  }
375  }
376 }
377 
378 /// AddSSAUpdateEntry - Add a definition and source virtual registers pair for
379 /// SSA update.
380 void TailDuplicatePass::AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg,
381  MachineBasicBlock *BB) {
382  DenseMap<unsigned, AvailableValsTy>::iterator LI= SSAUpdateVals.find(OrigReg);
383  if (LI != SSAUpdateVals.end())
384  LI->second.push_back(std::make_pair(BB, NewReg));
385  else {
386  AvailableValsTy Vals;
387  Vals.push_back(std::make_pair(BB, NewReg));
388  SSAUpdateVals.insert(std::make_pair(OrigReg, Vals));
389  SSAUpdateVRs.push_back(OrigReg);
390  }
391 }
392 
393 /// ProcessPHI - Process PHI node in TailBB by turning it into a copy in PredBB.
394 /// Remember the source register that's contributed by PredBB and update SSA
395 /// update map.
396 void TailDuplicatePass::ProcessPHI(
398  DenseMap<unsigned, unsigned> &LocalVRMap,
399  SmallVectorImpl<std::pair<unsigned, unsigned> > &Copies,
400  const DenseSet<unsigned> &RegsUsedByPhi, bool Remove) {
401  unsigned DefReg = MI->getOperand(0).getReg();
402  unsigned SrcOpIdx = getPHISrcRegOpIdx(MI, PredBB);
403  assert(SrcOpIdx && "Unable to find matching PHI source?");
404  unsigned SrcReg = MI->getOperand(SrcOpIdx).getReg();
405  const TargetRegisterClass *RC = MRI->getRegClass(DefReg);
406  LocalVRMap.insert(std::make_pair(DefReg, SrcReg));
407 
408  // Insert a copy from source to the end of the block. The def register is the
409  // available value liveout of the block.
410  unsigned NewDef = MRI->createVirtualRegister(RC);
411  Copies.push_back(std::make_pair(NewDef, SrcReg));
412  if (isDefLiveOut(DefReg, TailBB, MRI) || RegsUsedByPhi.count(DefReg))
413  AddSSAUpdateEntry(DefReg, NewDef, PredBB);
414 
415  if (!Remove)
416  return;
417 
418  // Remove PredBB from the PHI node.
419  MI->RemoveOperand(SrcOpIdx+1);
420  MI->RemoveOperand(SrcOpIdx);
421  if (MI->getNumOperands() == 1)
422  MI->eraseFromParent();
423 }
424 
425 /// DuplicateInstruction - Duplicate a TailBB instruction to PredBB and update
426 /// the source operands due to earlier PHI translation.
427 void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI,
428  MachineBasicBlock *TailBB,
429  MachineBasicBlock *PredBB,
430  MachineFunction &MF,
431  DenseMap<unsigned, unsigned> &LocalVRMap,
432  const DenseSet<unsigned> &UsedByPhi) {
433  MachineInstr *NewMI = TII->duplicate(MI, MF);
434  for (unsigned i = 0, e = NewMI->getNumOperands(); i != e; ++i) {
435  MachineOperand &MO = NewMI->getOperand(i);
436  if (!MO.isReg())
437  continue;
438  unsigned Reg = MO.getReg();
440  continue;
441  if (MO.isDef()) {
442  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
443  unsigned NewReg = MRI->createVirtualRegister(RC);
444  MO.setReg(NewReg);
445  LocalVRMap.insert(std::make_pair(Reg, NewReg));
446  if (isDefLiveOut(Reg, TailBB, MRI) || UsedByPhi.count(Reg))
447  AddSSAUpdateEntry(Reg, NewReg, PredBB);
448  } else {
450  if (VI != LocalVRMap.end()) {
451  MO.setReg(VI->second);
452  // Clear any kill flags from this operand. The new register could have
453  // uses after this one, so kills are not valid here.
454  MO.setIsKill(false);
455  MRI->constrainRegClass(VI->second, MRI->getRegClass(Reg));
456  }
457  }
458  }
459  PredBB->insert(PredBB->instr_end(), NewMI);
460 }
461 
462 /// UpdateSuccessorsPHIs - After FromBB is tail duplicated into its predecessor
463 /// blocks, the successors have gained new predecessors. Update the PHI
464 /// instructions in them accordingly.
465 void
466 TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead,
470  SE = Succs.end(); SI != SE; ++SI) {
471  MachineBasicBlock *SuccBB = *SI;
472  for (MachineBasicBlock::iterator II = SuccBB->begin(), EE = SuccBB->end();
473  II != EE; ++II) {
474  if (!II->isPHI())
475  break;
476  MachineInstrBuilder MIB(*FromBB->getParent(), II);
477  unsigned Idx = 0;
478  for (unsigned i = 1, e = II->getNumOperands(); i != e; i += 2) {
479  MachineOperand &MO = II->getOperand(i+1);
480  if (MO.getMBB() == FromBB) {
481  Idx = i;
482  break;
483  }
484  }
485 
486  assert(Idx != 0);
487  MachineOperand &MO0 = II->getOperand(Idx);
488  unsigned Reg = MO0.getReg();
489  if (isDead) {
490  // Folded into the previous BB.
491  // There could be duplicate phi source entries. FIXME: Should sdisel
492  // or earlier pass fixed this?
493  for (unsigned i = II->getNumOperands()-2; i != Idx; i -= 2) {
494  MachineOperand &MO = II->getOperand(i+1);
495  if (MO.getMBB() == FromBB) {
496  II->RemoveOperand(i+1);
497  II->RemoveOperand(i);
498  }
499  }
500  } else
501  Idx = 0;
502 
503  // If Idx is set, the operands at Idx and Idx+1 must be removed.
504  // We reuse the location to avoid expensive RemoveOperand calls.
505 
507  if (LI != SSAUpdateVals.end()) {
508  // This register is defined in the tail block.
509  for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
510  MachineBasicBlock *SrcBB = LI->second[j].first;
511  // If we didn't duplicate a bb into a particular predecessor, we
512  // might still have added an entry to SSAUpdateVals to correcly
513  // recompute SSA. If that case, avoid adding a dummy extra argument
514  // this PHI.
515  if (!SrcBB->isSuccessor(SuccBB))
516  continue;
517 
518  unsigned SrcReg = LI->second[j].second;
519  if (Idx != 0) {
520  II->getOperand(Idx).setReg(SrcReg);
521  II->getOperand(Idx+1).setMBB(SrcBB);
522  Idx = 0;
523  } else {
524  MIB.addReg(SrcReg).addMBB(SrcBB);
525  }
526  }
527  } else {
528  // Live in tail block, must also be live in predecessors.
529  for (unsigned j = 0, ee = TDBBs.size(); j != ee; ++j) {
530  MachineBasicBlock *SrcBB = TDBBs[j];
531  if (Idx != 0) {
532  II->getOperand(Idx).setReg(Reg);
533  II->getOperand(Idx+1).setMBB(SrcBB);
534  Idx = 0;
535  } else {
536  MIB.addReg(Reg).addMBB(SrcBB);
537  }
538  }
539  }
540  if (Idx != 0) {
541  II->RemoveOperand(Idx+1);
542  II->RemoveOperand(Idx);
543  }
544  }
545  }
546 }
547 
548 /// shouldTailDuplicate - Determine if it is profitable to duplicate this block.
549 bool
550 TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF,
551  bool IsSimple,
552  MachineBasicBlock &TailBB) {
553  // Only duplicate blocks that end with unconditional branches.
554  if (TailBB.canFallThrough())
555  return false;
556 
557  // Don't try to tail-duplicate single-block loops.
558  if (TailBB.isSuccessor(&TailBB))
559  return false;
560 
561  // Set the limit on the cost to duplicate. When optimizing for size,
562  // duplicate only one, because one branch instruction can be eliminated to
563  // compensate for the duplication.
564  unsigned MaxDuplicateCount;
565  if (TailDuplicateSize.getNumOccurrences() == 0 &&
567  MaxDuplicateCount = 1;
568  else
569  MaxDuplicateCount = TailDuplicateSize;
570 
571  // If the target has hardware branch prediction that can handle indirect
572  // branches, duplicating them can often make them predictable when there
573  // are common paths through the code. The limit needs to be high enough
574  // to allow undoing the effects of tail merging and other optimizations
575  // that rearrange the predecessors of the indirect branch.
576 
577  bool HasIndirectbr = false;
578  if (!TailBB.empty())
579  HasIndirectbr = TailBB.back().isIndirectBranch();
580 
581  if (HasIndirectbr && PreRegAlloc)
582  MaxDuplicateCount = 20;
583 
584  // Check the instructions in the block to determine whether tail-duplication
585  // is invalid or unlikely to be profitable.
586  unsigned InstrCount = 0;
587  for (MachineBasicBlock::iterator I = TailBB.begin(); I != TailBB.end(); ++I) {
588  // Non-duplicable things shouldn't be tail-duplicated.
589  if (I->isNotDuplicable())
590  return false;
591 
592  // Do not duplicate 'return' instructions if this is a pre-regalloc run.
593  // A return may expand into a lot more instructions (e.g. reload of callee
594  // saved registers) after PEI.
595  if (PreRegAlloc && I->isReturn())
596  return false;
597 
598  // Avoid duplicating calls before register allocation. Calls presents a
599  // barrier to register allocation so duplicating them may end up increasing
600  // spills.
601  if (PreRegAlloc && I->isCall())
602  return false;
603 
604  if (!I->isPHI() && !I->isDebugValue())
605  InstrCount += 1;
606 
607  if (InstrCount > MaxDuplicateCount)
608  return false;
609  }
610 
611  if (HasIndirectbr && PreRegAlloc)
612  return true;
613 
614  if (IsSimple)
615  return true;
616 
617  if (!PreRegAlloc)
618  return true;
619 
620  return canCompletelyDuplicateBB(TailBB);
621 }
622 
623 /// isSimpleBB - True if this BB has only one unconditional jump.
624 bool
625 TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) {
626  if (TailBB->succ_size() != 1)
627  return false;
628  if (TailBB->pred_empty())
629  return false;
631  if (I == TailBB->end())
632  return true;
633  return I->isUnconditionalBranch();
634 }
635 
636 static bool
640  SE = A.succ_end(); SI != SE; ++SI) {
641  MachineBasicBlock *BB = *SI;
642  if (SuccsB.count(BB) && !BB->empty() && BB->begin()->isPHI())
643  return true;
644  }
645 
646  return false;
647 }
648 
649 bool
650 TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) {
652  PE = BB.pred_end(); PI != PE; ++PI) {
653  MachineBasicBlock *PredBB = *PI;
654 
655  if (PredBB->succ_size() > 1)
656  return false;
657 
658  MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
660  if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
661  return false;
662 
663  if (!PredCond.empty())
664  return false;
665  }
666  return true;
667 }
668 
669 bool
670 TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB,
672  const DenseSet<unsigned> &UsedByPhi,
675  TailBB->succ_end());
677  TailBB->pred_end());
678  bool Changed = false;
680  PE = Preds.end(); PI != PE; ++PI) {
681  MachineBasicBlock *PredBB = *PI;
682 
683  if (PredBB->getLandingPadSuccessor())
684  continue;
685 
686  if (bothUsedInPHI(*PredBB, Succs))
687  continue;
688 
689  MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
691  if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
692  continue;
693 
694  Changed = true;
695  DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
696  << "From simple Succ: " << *TailBB);
697 
698  MachineBasicBlock *NewTarget = *TailBB->succ_begin();
699  MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(PredBB));
700 
701  // Make PredFBB explicit.
702  if (PredCond.empty())
703  PredFBB = PredTBB;
704 
705  // Make fall through explicit.
706  if (!PredTBB)
707  PredTBB = NextBB;
708  if (!PredFBB)
709  PredFBB = NextBB;
710 
711  // Redirect
712  if (PredFBB == TailBB)
713  PredFBB = NewTarget;
714  if (PredTBB == TailBB)
715  PredTBB = NewTarget;
716 
717  // Make the branch unconditional if possible
718  if (PredTBB == PredFBB) {
719  PredCond.clear();
720  PredFBB = nullptr;
721  }
722 
723  // Avoid adding fall through branches.
724  if (PredFBB == NextBB)
725  PredFBB = nullptr;
726  if (PredTBB == NextBB && PredFBB == nullptr)
727  PredTBB = nullptr;
728 
729  TII->RemoveBranch(*PredBB);
730 
731  if (PredTBB)
732  TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc());
733 
734  uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB);
735  PredBB->removeSuccessor(TailBB);
736  unsigned NumSuccessors = PredBB->succ_size();
737  assert(NumSuccessors <= 1);
738  if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
739  PredBB->addSuccessor(NewTarget, Weight);
740 
741  TDBBs.push_back(PredBB);
742  }
743  return Changed;
744 }
745 
746 /// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
747 /// of its predecessors.
748 bool
749 TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB,
750  bool IsSimple,
751  MachineFunction &MF,
754  DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
755 
756  DenseSet<unsigned> UsedByPhi;
757  getRegsUsedByPHIs(*TailBB, &UsedByPhi);
758 
759  if (IsSimple)
760  return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies);
761 
762  // Iterate through all the unique predecessors and tail-duplicate this
763  // block into them, if possible. Copying the list ahead of time also
764  // avoids trouble with the predecessor list reallocating.
765  bool Changed = false;
767  TailBB->pred_end());
769  PE = Preds.end(); PI != PE; ++PI) {
770  MachineBasicBlock *PredBB = *PI;
771 
772  assert(TailBB != PredBB &&
773  "Single-block loop should have been rejected earlier!");
774  // EH edges are ignored by AnalyzeBranch.
775  if (PredBB->succ_size() > 1)
776  continue;
777 
778  MachineBasicBlock *PredTBB, *PredFBB;
780  if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
781  continue;
782  if (!PredCond.empty())
783  continue;
784  // Don't duplicate into a fall-through predecessor (at least for now).
785  if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
786  continue;
787 
788  DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
789  << "From Succ: " << *TailBB);
790 
791  TDBBs.push_back(PredBB);
792 
793  // Remove PredBB's unconditional branch.
794  TII->RemoveBranch(*PredBB);
795 
796  if (RS && !TailBB->livein_empty()) {
797  // Update PredBB livein.
798  RS->enterBasicBlock(PredBB);
799  if (!PredBB->empty())
800  RS->forward(std::prev(PredBB->end()));
802  E = TailBB->livein_end(); I != E; ++I) {
803  if (!RS->isRegUsed(*I, false))
804  // If a register is previously livein to the tail but it's not live
805  // at the end of predecessor BB, then it should be added to its
806  // livein list.
807  PredBB->addLiveIn(*I);
808  }
809  }
810 
811  // Clone the contents of TailBB into PredBB.
812  DenseMap<unsigned, unsigned> LocalVRMap;
814  // Use instr_iterator here to properly handle bundles, e.g.
815  // ARM Thumb2 IT block.
817  while (I != TailBB->instr_end()) {
818  MachineInstr *MI = &*I;
819  ++I;
820  if (MI->isPHI()) {
821  // Replace the uses of the def of the PHI with the register coming
822  // from PredBB.
823  ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
824  } else {
825  // Replace def of virtual registers with new registers, and update
826  // uses with PHI source register or the new registers.
827  DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi);
828  }
829  }
831  for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
832  Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
833  TII->get(TargetOpcode::COPY),
834  CopyInfos[i].first).addReg(CopyInfos[i].second));
835  }
836 
837  // Simplify
838  TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true);
839 
840  NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
841 
842  // Update the CFG.
843  PredBB->removeSuccessor(PredBB->succ_begin());
844  assert(PredBB->succ_empty() &&
845  "TailDuplicate called on block with multiple successors!");
846  for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
847  E = TailBB->succ_end(); I != E; ++I)
848  PredBB->addSuccessor(*I, MBPI->getEdgeWeight(TailBB, I));
849 
850  Changed = true;
851  ++NumTailDups;
852  }
853 
854  // If TailBB was duplicated into all its predecessors except for the prior
855  // block, which falls through unconditionally, move the contents of this
856  // block into the prior block.
857  MachineBasicBlock *PrevBB = std::prev(MachineFunction::iterator(TailBB));
858  MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr;
860  // This has to check PrevBB->succ_size() because EH edges are ignored by
861  // AnalyzeBranch.
862  if (PrevBB->succ_size() == 1 &&
863  !TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) &&
864  PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 &&
865  !TailBB->hasAddressTaken()) {
866  DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
867  << "From MBB: " << *TailBB);
868  if (PreRegAlloc) {
869  DenseMap<unsigned, unsigned> LocalVRMap;
871  MachineBasicBlock::iterator I = TailBB->begin();
872  // Process PHI instructions first.
873  while (I != TailBB->end() && I->isPHI()) {
874  // Replace the uses of the def of the PHI with the register coming
875  // from PredBB.
876  MachineInstr *MI = &*I++;
877  ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
878  if (MI->getParent())
879  MI->eraseFromParent();
880  }
881 
882  // Now copy the non-PHI instructions.
883  while (I != TailBB->end()) {
884  // Replace def of virtual registers with new registers, and update
885  // uses with PHI source register or the new registers.
886  MachineInstr *MI = &*I++;
887  assert(!MI->isBundle() && "Not expecting bundles before regalloc!");
888  DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi);
889  MI->eraseFromParent();
890  }
892  for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
893  Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
894  TII->get(TargetOpcode::COPY),
895  CopyInfos[i].first)
896  .addReg(CopyInfos[i].second));
897  }
898  } else {
899  // No PHIs to worry about, just splice the instructions over.
900  PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
901  }
902  PrevBB->removeSuccessor(PrevBB->succ_begin());
903  assert(PrevBB->succ_empty());
904  PrevBB->transferSuccessors(TailBB);
905  TDBBs.push_back(PrevBB);
906  Changed = true;
907  }
908 
909  // If this is after register allocation, there are no phis to fix.
910  if (!PreRegAlloc)
911  return Changed;
912 
913  // If we made no changes so far, we are safe.
914  if (!Changed)
915  return Changed;
916 
917 
918  // Handle the nasty case in that we duplicated a block that is part of a loop
919  // into some but not all of its predecessors. For example:
920  // 1 -> 2 <-> 3 |
921  // \ |
922  // \---> rest |
923  // if we duplicate 2 into 1 but not into 3, we end up with
924  // 12 -> 3 <-> 2 -> rest |
925  // \ / |
926  // \----->-----/ |
927  // If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
928  // with a phi in 3 (which now dominates 2).
929  // What we do here is introduce a copy in 3 of the register defined by the
930  // phi, just like when we are duplicating 2 into 3, but we don't copy any
931  // real instructions or remove the 3 -> 2 edge from the phi in 2.
933  PE = Preds.end(); PI != PE; ++PI) {
934  MachineBasicBlock *PredBB = *PI;
935  if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end())
936  continue;
937 
938  // EH edges
939  if (PredBB->succ_size() != 1)
940  continue;
941 
942  DenseMap<unsigned, unsigned> LocalVRMap;
944  MachineBasicBlock::iterator I = TailBB->begin();
945  // Process PHI instructions first.
946  while (I != TailBB->end() && I->isPHI()) {
947  // Replace the uses of the def of the PHI with the register coming
948  // from PredBB.
949  MachineInstr *MI = &*I++;
950  ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
951  }
953  for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
954  Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
955  TII->get(TargetOpcode::COPY),
956  CopyInfos[i].first).addReg(CopyInfos[i].second));
957  }
958  }
959 
960  return Changed;
961 }
962 
963 /// RemoveDeadBlock - Remove the specified dead machine basic block from the
964 /// function, updating the CFG.
965 void TailDuplicatePass::RemoveDeadBlock(MachineBasicBlock *MBB) {
966  assert(MBB->pred_empty() && "MBB must be dead!");
967  DEBUG(dbgs() << "\nRemoving MBB: " << *MBB);
968 
969  // Remove all successors.
970  while (!MBB->succ_empty())
971  MBB->removeSuccessor(MBB->succ_end()-1);
972 
973  // Remove the block.
974  MBB->eraseFromParent();
975 }
unsigned succ_size() const
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
static cl::opt< bool > TailDupVerify("tail-dup-verify", cl::desc("Verify sanity of PHI instructions during taildup"), cl::init(false), cl::Hidden)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
instr_iterator instr_begin()
instr_iterator instr_end()
STATISTIC(NumFunctions,"Total number of functions")
static void Found()
MachineBasicBlock * getMBB() const
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
DenseSet - This implements a dense probed hash-table based set.
Definition: DenseSet.h:39
iterator getFirstNonDebugInstr()
getFirstNonDebugInstr - returns an iterator to the first non-debug instruction in the basic block...
std::vector< unsigned >::const_iterator livein_iterator
iterator getFirstTerminator()
getFirstTerminator - returns an iterator to the first terminator instruction of this basic block...
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:276
void addLiveIn(unsigned Reg)
Adds the specified register as a live in.
A debug info location.
Definition: DebugLoc.h:34
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB)
Instructions::iterator instr_iterator
iterator end()
Get an iterator to the end of the SetVector.
Definition: SetVector.h:79
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
COPY - Target-independent register copy.
Definition: TargetOpcodes.h:86
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:169
livein_iterator livein_begin() const
AnalysisUsage & addRequired()
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing function and deletes it...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
bool isPHI() const
Definition: MachineInstr.h:757
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
void RemoveOperand(unsigned i)
Erase an operand from an instruction, leaving it with one fewer operand than it started with...
const HexagonRegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
void transferSuccessors(MachineBasicBlock *fromMBB)
transferSuccessors - Transfers all the successors from MBB to this machine basic block (i...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
bool canFallThrough()
canFallThrough - Return true if the block can implicitly transfer control to the block after it by fa...
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:69
std::vector< MachineBasicBlock * >::iterator pred_iterator
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
static bool bothUsedInPHI(const MachineBasicBlock &A, SmallPtrSet< MachineBasicBlock *, 8 > SuccsB)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:748
bool isBundle() const
Definition: MachineInstr.h:775
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
static void getRegsUsedByPHIs(const MachineBasicBlock &BB, DenseSet< unsigned > *UsedByPhi)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned RemoveBranch(MachineBasicBlock &MBB) const override
size_type count(const ValueT &V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:65
livein_iterator livein_end() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
static cl::opt< unsigned > TailDupLimit("tail-dup-limit", cl::init(~0U), cl::Hidden)
bool isIndirectBranch(QueryType Type=AnyInBundle) const
Return true if this is an indirect branch, such as a branch through a register.
Definition: MachineInstr.h:433
bool isCopy() const
Definition: MachineInstr.h:778
Represent the analysis usage information of a pass.
const MachineBasicBlock * getLandingPadSuccessor() const
getLandingPadSuccessor - If this block has a successor that is a landing pad, return it...
static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, const MachineRegisterInfo *MRI)
static cl::opt< unsigned > TailDuplicateSize("tail-dup-size", cl::desc("Maximum instructions to consider tail duplicating"), cl::init(2), cl::Hidden)
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
void removeSuccessor(MachineBasicBlock *succ)
removeSuccessor - Remove successor from the successors list of this MachineBasicBlock.
iterator_range< use_instr_iterator > use_instructions(unsigned Reg) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void setIsKill(bool Val=true)
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:217
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:147
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
std::vector< std::pair< MachineBasicBlock *, unsigned > > AvailableValsTy
MachineOperand class - Representation of each machine instruction operand.
char & TailDuplicateID
TailDuplicate - Duplicate blocks with unconditional branches into tails of their predecessors.
SI Lower i1 Copies
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:51
bool hasAddressTaken() const
hasAddressTaken - Test whether this block is potentially the target of an indirect branch...
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:217
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:54
INITIALIZE_PASS(TailDuplicatePass,"tailduplication","Tail Duplication", false, false) bool TailDuplicatePass
iterator find(const KeyT &Val)
Definition: DenseMap.h:124
static void VerifyPHIs(MachineFunction &MF, bool CheckExtra)
unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, DebugLoc DL) const override
unsigned getReg() const
getReg - Returns the register number.
std::vector< MachineBasicBlock * >::const_iterator const_succ_iterator
BasicBlockListType::iterator iterator
#define DEBUG(X)
Definition: Debug.h:92
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
isLayoutSuccessor - Return true if the specified MBB will be emitted immediately after this block...
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
void addSuccessor(MachineBasicBlock *succ, uint32_t weight=0)
addSuccessor - Add succ as a successor of this MachineBasicBlock.
unsigned pred_size() const
MachineModuleInfo - This class contains meta information specific to a module.