LLVM  4.0.0
IfConversion.cpp
Go to the documentation of this file.
1 //===-- IfConversion.cpp - Machine code if conversion pass. ---------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the machine instruction level if-conversion pass, which
11 // tries to convert conditional branches into predicated instructions.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/CodeGen/Passes.h"
16 #include "BranchFolding.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/ScopeExit.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/Statistic.h"
30 #include "llvm/Support/Debug.h"
37 #include <algorithm>
38 #include <utility>
39 
40 using namespace llvm;
41 
42 #define DEBUG_TYPE "ifcvt"
43 
44 // Hidden options for help debugging.
45 static cl::opt<int> IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden);
46 static cl::opt<int> IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden);
47 static cl::opt<int> IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden);
48 static cl::opt<bool> DisableSimple("disable-ifcvt-simple",
49  cl::init(false), cl::Hidden);
50 static cl::opt<bool> DisableSimpleF("disable-ifcvt-simple-false",
51  cl::init(false), cl::Hidden);
52 static cl::opt<bool> DisableTriangle("disable-ifcvt-triangle",
53  cl::init(false), cl::Hidden);
54 static cl::opt<bool> DisableTriangleR("disable-ifcvt-triangle-rev",
55  cl::init(false), cl::Hidden);
56 static cl::opt<bool> DisableTriangleF("disable-ifcvt-triangle-false",
57  cl::init(false), cl::Hidden);
58 static cl::opt<bool> DisableTriangleFR("disable-ifcvt-triangle-false-rev",
59  cl::init(false), cl::Hidden);
60 static cl::opt<bool> DisableDiamond("disable-ifcvt-diamond",
61  cl::init(false), cl::Hidden);
62 static cl::opt<bool> DisableForkedDiamond("disable-ifcvt-forked-diamond",
63  cl::init(false), cl::Hidden);
64 static cl::opt<bool> IfCvtBranchFold("ifcvt-branch-fold",
65  cl::init(true), cl::Hidden);
66 
67 STATISTIC(NumSimple, "Number of simple if-conversions performed");
68 STATISTIC(NumSimpleFalse, "Number of simple (F) if-conversions performed");
69 STATISTIC(NumTriangle, "Number of triangle if-conversions performed");
70 STATISTIC(NumTriangleRev, "Number of triangle (R) if-conversions performed");
71 STATISTIC(NumTriangleFalse,"Number of triangle (F) if-conversions performed");
72 STATISTIC(NumTriangleFRev, "Number of triangle (F/R) if-conversions performed");
73 STATISTIC(NumDiamonds, "Number of diamond if-conversions performed");
74 STATISTIC(NumForkedDiamonds, "Number of forked-diamond if-conversions performed");
75 STATISTIC(NumIfConvBBs, "Number of if-converted blocks");
76 STATISTIC(NumDupBBs, "Number of duplicated blocks");
77 STATISTIC(NumUnpred, "Number of true blocks of diamonds unpredicated");
78 
79 namespace {
80  class IfConverter : public MachineFunctionPass {
81  enum IfcvtKind {
82  ICNotClassfied, // BB data valid, but not classified.
83  ICSimpleFalse, // Same as ICSimple, but on the false path.
84  ICSimple, // BB is entry of an one split, no rejoin sub-CFG.
85  ICTriangleFRev, // Same as ICTriangleFalse, but false path rev condition.
86  ICTriangleRev, // Same as ICTriangle, but true path rev condition.
87  ICTriangleFalse, // Same as ICTriangle, but on the false path.
88  ICTriangle, // BB is entry of a triangle sub-CFG.
89  ICDiamond, // BB is entry of a diamond sub-CFG.
90  ICForkedDiamond // BB is entry of an almost diamond sub-CFG, with a
91  // common tail that can be shared.
92  };
93 
94  /// One per MachineBasicBlock, this is used to cache the result
95  /// if-conversion feasibility analysis. This includes results from
96  /// TargetInstrInfo::analyzeBranch() (i.e. TBB, FBB, and Cond), and its
97  /// classification, and common tail block of its successors (if it's a
98  /// diamond shape), its size, whether it's predicable, and whether any
99  /// instruction can clobber the 'would-be' predicate.
100  ///
101  /// IsDone - True if BB is not to be considered for ifcvt.
102  /// IsBeingAnalyzed - True if BB is currently being analyzed.
103  /// IsAnalyzed - True if BB has been analyzed (info is still valid).
104  /// IsEnqueued - True if BB has been enqueued to be ifcvt'ed.
105  /// IsBrAnalyzable - True if analyzeBranch() returns false.
106  /// HasFallThrough - True if BB may fallthrough to the following BB.
107  /// IsUnpredicable - True if BB is known to be unpredicable.
108  /// ClobbersPred - True if BB could modify predicates (e.g. has
109  /// cmp, call, etc.)
110  /// NonPredSize - Number of non-predicated instructions.
111  /// ExtraCost - Extra cost for multi-cycle instructions.
112  /// ExtraCost2 - Some instructions are slower when predicated
113  /// BB - Corresponding MachineBasicBlock.
114  /// TrueBB / FalseBB- See analyzeBranch().
115  /// BrCond - Conditions for end of block conditional branches.
116  /// Predicate - Predicate used in the BB.
117  struct BBInfo {
118  bool IsDone : 1;
119  bool IsBeingAnalyzed : 1;
120  bool IsAnalyzed : 1;
121  bool IsEnqueued : 1;
122  bool IsBrAnalyzable : 1;
123  bool IsBrReversible : 1;
124  bool HasFallThrough : 1;
125  bool IsUnpredicable : 1;
126  bool CannotBeCopied : 1;
127  bool ClobbersPred : 1;
128  unsigned NonPredSize;
129  unsigned ExtraCost;
130  unsigned ExtraCost2;
131  MachineBasicBlock *BB;
132  MachineBasicBlock *TrueBB;
133  MachineBasicBlock *FalseBB;
136  BBInfo() : IsDone(false), IsBeingAnalyzed(false),
137  IsAnalyzed(false), IsEnqueued(false), IsBrAnalyzable(false),
138  IsBrReversible(false), HasFallThrough(false),
139  IsUnpredicable(false), CannotBeCopied(false),
140  ClobbersPred(false), NonPredSize(0), ExtraCost(0),
141  ExtraCost2(0), BB(nullptr), TrueBB(nullptr),
142  FalseBB(nullptr) {}
143  };
144 
145  /// Record information about pending if-conversions to attempt:
146  /// BBI - Corresponding BBInfo.
147  /// Kind - Type of block. See IfcvtKind.
148  /// NeedSubsumption - True if the to-be-predicated BB has already been
149  /// predicated.
150  /// NumDups - Number of instructions that would be duplicated due
151  /// to this if-conversion. (For diamonds, the number of
152  /// identical instructions at the beginnings of both
153  /// paths).
154  /// NumDups2 - For diamonds, the number of identical instructions
155  /// at the ends of both paths.
156  struct IfcvtToken {
157  BBInfo &BBI;
158  IfcvtKind Kind;
159  unsigned NumDups;
160  unsigned NumDups2;
161  bool NeedSubsumption : 1;
162  bool TClobbersPred : 1;
163  bool FClobbersPred : 1;
164  IfcvtToken(BBInfo &b, IfcvtKind k, bool s, unsigned d, unsigned d2 = 0,
165  bool tc = false, bool fc = false)
166  : BBI(b), Kind(k), NumDups(d), NumDups2(d2), NeedSubsumption(s),
167  TClobbersPred(tc), FClobbersPred(fc) {}
168  };
169 
170  /// Results of if-conversion feasibility analysis indexed by basic block
171  /// number.
172  std::vector<BBInfo> BBAnalysis;
173  TargetSchedModel SchedModel;
174 
175  const TargetLoweringBase *TLI;
176  const TargetInstrInfo *TII;
177  const TargetRegisterInfo *TRI;
178  const MachineBranchProbabilityInfo *MBPI;
180 
181  LivePhysRegs Redefs;
182  LivePhysRegs DontKill;
183 
184  bool PreRegAlloc;
185  bool MadeChange;
186  int FnNum;
187  std::function<bool(const MachineFunction &)> PredicateFtor;
188 
189  public:
190  static char ID;
191  IfConverter(std::function<bool(const MachineFunction &)> Ftor = nullptr)
192  : MachineFunctionPass(ID), FnNum(-1), PredicateFtor(std::move(Ftor)) {
194  }
195 
196  void getAnalysisUsage(AnalysisUsage &AU) const override {
200  }
201 
202  bool runOnMachineFunction(MachineFunction &MF) override;
203 
204  MachineFunctionProperties getRequiredProperties() const override {
207  }
208 
209  private:
210  bool reverseBranchCondition(BBInfo &BBI) const;
211  bool ValidSimple(BBInfo &TrueBBI, unsigned &Dups,
212  BranchProbability Prediction) const;
213  bool ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
214  bool FalseBranch, unsigned &Dups,
215  BranchProbability Prediction) const;
216  bool CountDuplicatedInstructions(
219  unsigned &Dups1, unsigned &Dups2,
221  bool SkipUnconditionalBranches) const;
222  bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
223  unsigned &Dups1, unsigned &Dups2,
224  BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const;
225  bool ValidForkedDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
226  unsigned &Dups1, unsigned &Dups2,
227  BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const;
228  void AnalyzeBranches(BBInfo &BBI);
229  void ScanInstructions(BBInfo &BBI,
232  bool BranchUnpredicable = false) const;
233  bool RescanInstructions(
236  BBInfo &TrueBBI, BBInfo &FalseBBI) const;
237  void AnalyzeBlock(MachineBasicBlock &MBB,
238  std::vector<std::unique_ptr<IfcvtToken>> &Tokens);
239  bool FeasibilityAnalysis(BBInfo &BBI, SmallVectorImpl<MachineOperand> &Cond,
240  bool isTriangle = false, bool RevBranch = false,
241  bool hasCommonTail = false);
242  void AnalyzeBlocks(MachineFunction &MF,
243  std::vector<std::unique_ptr<IfcvtToken>> &Tokens);
244  void InvalidatePreds(MachineBasicBlock &MBB);
245  void RemoveExtraEdges(BBInfo &BBI);
246  bool IfConvertSimple(BBInfo &BBI, IfcvtKind Kind);
247  bool IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind);
248  bool IfConvertDiamondCommon(BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI,
249  unsigned NumDups1, unsigned NumDups2,
250  bool TClobbersPred, bool FClobbersPred,
251  bool RemoveBranch, bool MergeAddEdges);
252  bool IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
253  unsigned NumDups1, unsigned NumDups2,
254  bool TClobbers, bool FClobbers);
255  bool IfConvertForkedDiamond(BBInfo &BBI, IfcvtKind Kind,
256  unsigned NumDups1, unsigned NumDups2,
257  bool TClobbers, bool FClobbers);
258  void PredicateBlock(BBInfo &BBI,
261  SmallSet<unsigned, 4> *LaterRedefs = nullptr);
262  void CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
264  bool IgnoreBr = false);
265  void MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges = true);
266 
267  bool MeetIfcvtSizeLimit(MachineBasicBlock &BB,
268  unsigned Cycle, unsigned Extra,
269  BranchProbability Prediction) const {
270  return Cycle > 0 && TII->isProfitableToIfCvt(BB, Cycle, Extra,
271  Prediction);
272  }
273 
274  bool MeetIfcvtSizeLimit(MachineBasicBlock &TBB,
275  unsigned TCycle, unsigned TExtra,
276  MachineBasicBlock &FBB,
277  unsigned FCycle, unsigned FExtra,
278  BranchProbability Prediction) const {
279  return TCycle > 0 && FCycle > 0 &&
280  TII->isProfitableToIfCvt(TBB, TCycle, TExtra, FBB, FCycle, FExtra,
281  Prediction);
282  }
283 
284  /// Returns true if Block ends without a terminator.
285  bool blockAlwaysFallThrough(BBInfo &BBI) const {
286  return BBI.IsBrAnalyzable && BBI.TrueBB == nullptr;
287  }
288 
289  /// Used to sort if-conversion candidates.
290  static bool IfcvtTokenCmp(const std::unique_ptr<IfcvtToken> &C1,
291  const std::unique_ptr<IfcvtToken> &C2) {
292  int Incr1 = (C1->Kind == ICDiamond)
293  ? -(int)(C1->NumDups + C1->NumDups2) : (int)C1->NumDups;
294  int Incr2 = (C2->Kind == ICDiamond)
295  ? -(int)(C2->NumDups + C2->NumDups2) : (int)C2->NumDups;
296  if (Incr1 > Incr2)
297  return true;
298  else if (Incr1 == Incr2) {
299  // Favors subsumption.
300  if (!C1->NeedSubsumption && C2->NeedSubsumption)
301  return true;
302  else if (C1->NeedSubsumption == C2->NeedSubsumption) {
303  // Favors diamond over triangle, etc.
304  if ((unsigned)C1->Kind < (unsigned)C2->Kind)
305  return true;
306  else if (C1->Kind == C2->Kind)
307  return C1->BBI.BB->getNumber() < C2->BBI.BB->getNumber();
308  }
309  }
310  return false;
311  }
312  };
313 
314  char IfConverter::ID = 0;
315 }
316 
318 
319 INITIALIZE_PASS_BEGIN(IfConverter, "if-converter", "If Converter", false, false)
321 INITIALIZE_PASS_END(IfConverter, "if-converter", "If Converter", false, false)
322 
323 bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
324  if (skipFunction(*MF.getFunction()) || (PredicateFtor && !PredicateFtor(MF)))
325  return false;
326 
327  const TargetSubtargetInfo &ST = MF.getSubtarget();
328  TLI = ST.getTargetLowering();
329  TII = ST.getInstrInfo();
330  TRI = ST.getRegisterInfo();
331  BranchFolder::MBFIWrapper MBFI(getAnalysis<MachineBlockFrequencyInfo>());
332  MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
333  MRI = &MF.getRegInfo();
334  SchedModel.init(ST.getSchedModel(), &ST, TII);
335 
336  if (!TII) return false;
337 
338  PreRegAlloc = MRI->isSSA();
339 
340  bool BFChange = false;
341  if (!PreRegAlloc) {
342  // Tail merge tend to expose more if-conversion opportunities.
343  BranchFolder BF(true, false, MBFI, *MBPI);
344  BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo(),
345  getAnalysisIfAvailable<MachineModuleInfo>());
346  }
347 
348  DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'"
349  << MF.getName() << "\'");
350 
351  if (FnNum < IfCvtFnStart || (IfCvtFnStop != -1 && FnNum > IfCvtFnStop)) {
352  DEBUG(dbgs() << " skipped\n");
353  return false;
354  }
355  DEBUG(dbgs() << "\n");
356 
357  MF.RenumberBlocks();
358  BBAnalysis.resize(MF.getNumBlockIDs());
359 
360  std::vector<std::unique_ptr<IfcvtToken>> Tokens;
361  MadeChange = false;
362  unsigned NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle +
363  NumTriangleRev + NumTriangleFalse + NumTriangleFRev + NumDiamonds;
364  while (IfCvtLimit == -1 || (int)NumIfCvts < IfCvtLimit) {
365  // Do an initial analysis for each basic block and find all the potential
366  // candidates to perform if-conversion.
367  bool Change = false;
368  AnalyzeBlocks(MF, Tokens);
369  while (!Tokens.empty()) {
370  std::unique_ptr<IfcvtToken> Token = std::move(Tokens.back());
371  Tokens.pop_back();
372  BBInfo &BBI = Token->BBI;
373  IfcvtKind Kind = Token->Kind;
374  unsigned NumDups = Token->NumDups;
375  unsigned NumDups2 = Token->NumDups2;
376 
377  // If the block has been evicted out of the queue or it has already been
378  // marked dead (due to it being predicated), then skip it.
379  if (BBI.IsDone)
380  BBI.IsEnqueued = false;
381  if (!BBI.IsEnqueued)
382  continue;
383 
384  BBI.IsEnqueued = false;
385 
386  bool RetVal = false;
387  switch (Kind) {
388  default: llvm_unreachable("Unexpected!");
389  case ICSimple:
390  case ICSimpleFalse: {
391  bool isFalse = Kind == ICSimpleFalse;
392  if ((isFalse && DisableSimpleF) || (!isFalse && DisableSimple)) break;
393  DEBUG(dbgs() << "Ifcvt (Simple" << (Kind == ICSimpleFalse ?
394  " false" : "")
395  << "): BB#" << BBI.BB->getNumber() << " ("
396  << ((Kind == ICSimpleFalse)
397  ? BBI.FalseBB->getNumber()
398  : BBI.TrueBB->getNumber()) << ") ");
399  RetVal = IfConvertSimple(BBI, Kind);
400  DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n");
401  if (RetVal) {
402  if (isFalse) ++NumSimpleFalse;
403  else ++NumSimple;
404  }
405  break;
406  }
407  case ICTriangle:
408  case ICTriangleRev:
409  case ICTriangleFalse:
410  case ICTriangleFRev: {
411  bool isFalse = Kind == ICTriangleFalse;
412  bool isRev = (Kind == ICTriangleRev || Kind == ICTriangleFRev);
413  if (DisableTriangle && !isFalse && !isRev) break;
414  if (DisableTriangleR && !isFalse && isRev) break;
415  if (DisableTriangleF && isFalse && !isRev) break;
416  if (DisableTriangleFR && isFalse && isRev) break;
417  DEBUG(dbgs() << "Ifcvt (Triangle");
418  if (isFalse)
419  DEBUG(dbgs() << " false");
420  if (isRev)
421  DEBUG(dbgs() << " rev");
422  DEBUG(dbgs() << "): BB#" << BBI.BB->getNumber() << " (T:"
423  << BBI.TrueBB->getNumber() << ",F:"
424  << BBI.FalseBB->getNumber() << ") ");
425  RetVal = IfConvertTriangle(BBI, Kind);
426  DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n");
427  if (RetVal) {
428  if (isFalse) {
429  if (isRev) ++NumTriangleFRev;
430  else ++NumTriangleFalse;
431  } else {
432  if (isRev) ++NumTriangleRev;
433  else ++NumTriangle;
434  }
435  }
436  break;
437  }
438  case ICDiamond: {
439  if (DisableDiamond) break;
440  DEBUG(dbgs() << "Ifcvt (Diamond): BB#" << BBI.BB->getNumber() << " (T:"
441  << BBI.TrueBB->getNumber() << ",F:"
442  << BBI.FalseBB->getNumber() << ") ");
443  RetVal = IfConvertDiamond(BBI, Kind, NumDups, NumDups2,
444  Token->TClobbersPred,
445  Token->FClobbersPred);
446  DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n");
447  if (RetVal) ++NumDiamonds;
448  break;
449  }
450  case ICForkedDiamond: {
451  if (DisableForkedDiamond) break;
452  DEBUG(dbgs() << "Ifcvt (Forked Diamond): BB#"
453  << BBI.BB->getNumber() << " (T:"
454  << BBI.TrueBB->getNumber() << ",F:"
455  << BBI.FalseBB->getNumber() << ") ");
456  RetVal = IfConvertForkedDiamond(BBI, Kind, NumDups, NumDups2,
457  Token->TClobbersPred,
458  Token->FClobbersPred);
459  DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n");
460  if (RetVal) ++NumForkedDiamonds;
461  break;
462  }
463  }
464 
465  Change |= RetVal;
466 
467  NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + NumTriangleRev +
468  NumTriangleFalse + NumTriangleFRev + NumDiamonds;
469  if (IfCvtLimit != -1 && (int)NumIfCvts >= IfCvtLimit)
470  break;
471  }
472 
473  if (!Change)
474  break;
475  MadeChange |= Change;
476  }
477 
478  Tokens.clear();
479  BBAnalysis.clear();
480 
481  if (MadeChange && IfCvtBranchFold) {
482  BranchFolder BF(false, false, MBFI, *MBPI);
483  BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(),
484  getAnalysisIfAvailable<MachineModuleInfo>());
485  }
486 
487  MadeChange |= BFChange;
488  return MadeChange;
489 }
490 
491 /// BB has a fallthrough. Find its 'false' successor given its 'true' successor.
493  MachineBasicBlock *TrueBB) {
494  for (MachineBasicBlock *SuccBB : BB->successors()) {
495  if (SuccBB != TrueBB)
496  return SuccBB;
497  }
498  return nullptr;
499 }
500 
501 /// Reverse the condition of the end of the block branch. Swap block's 'true'
502 /// and 'false' successors.
503 bool IfConverter::reverseBranchCondition(BBInfo &BBI) const {
504  DebugLoc dl; // FIXME: this is nowhere
505  if (!TII->reverseBranchCondition(BBI.BrCond)) {
506  TII->removeBranch(*BBI.BB);
507  TII->insertBranch(*BBI.BB, BBI.FalseBB, BBI.TrueBB, BBI.BrCond, dl);
508  std::swap(BBI.TrueBB, BBI.FalseBB);
509  return true;
510  }
511  return false;
512 }
513 
514 /// Returns the next block in the function blocks ordering. If it is the end,
515 /// returns NULL.
519  if (++I == E)
520  return nullptr;
521  return &*I;
522 }
523 
524 /// Returns true if the 'true' block (along with its predecessor) forms a valid
525 /// simple shape for ifcvt. It also returns the number of instructions that the
526 /// ifcvt would need to duplicate if performed in Dups.
527 bool IfConverter::ValidSimple(BBInfo &TrueBBI, unsigned &Dups,
528  BranchProbability Prediction) const {
529  Dups = 0;
530  if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
531  return false;
532 
533  if (TrueBBI.IsBrAnalyzable)
534  return false;
535 
536  if (TrueBBI.BB->pred_size() > 1) {
537  if (TrueBBI.CannotBeCopied ||
538  !TII->isProfitableToDupForIfCvt(*TrueBBI.BB, TrueBBI.NonPredSize,
539  Prediction))
540  return false;
541  Dups = TrueBBI.NonPredSize;
542  }
543 
544  return true;
545 }
546 
547 /// Returns true if the 'true' and 'false' blocks (along with their common
548 /// predecessor) forms a valid triangle shape for ifcvt. If 'FalseBranch' is
549 /// true, it checks if 'true' block's false branch branches to the 'false' block
550 /// rather than the other way around. It also returns the number of instructions
551 /// that the ifcvt would need to duplicate if performed in 'Dups'.
552 bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
553  bool FalseBranch, unsigned &Dups,
554  BranchProbability Prediction) const {
555  Dups = 0;
556  if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
557  return false;
558 
559  if (TrueBBI.BB->pred_size() > 1) {
560  if (TrueBBI.CannotBeCopied)
561  return false;
562 
563  unsigned Size = TrueBBI.NonPredSize;
564  if (TrueBBI.IsBrAnalyzable) {
565  if (TrueBBI.TrueBB && TrueBBI.BrCond.empty())
566  // Ends with an unconditional branch. It will be removed.
567  --Size;
568  else {
569  MachineBasicBlock *FExit = FalseBranch
570  ? TrueBBI.TrueBB : TrueBBI.FalseBB;
571  if (FExit)
572  // Require a conditional branch
573  ++Size;
574  }
575  }
576  if (!TII->isProfitableToDupForIfCvt(*TrueBBI.BB, Size, Prediction))
577  return false;
578  Dups = Size;
579  }
580 
581  MachineBasicBlock *TExit = FalseBranch ? TrueBBI.FalseBB : TrueBBI.TrueBB;
582  if (!TExit && blockAlwaysFallThrough(TrueBBI)) {
583  MachineFunction::iterator I = TrueBBI.BB->getIterator();
584  if (++I == TrueBBI.BB->getParent()->end())
585  return false;
586  TExit = &*I;
587  }
588  return TExit && TExit == FalseBBI.BB;
589 }
590 
591 /// Shrink the provided inclusive range by one instruction.
592 /// If the range was one instruction (\p It == \p Begin), It is not modified,
593 /// but \p Empty is set to true.
594 static inline void shrinkInclusiveRange(
597  bool &Empty) {
598  if (It == Begin)
599  Empty = true;
600  else
601  It--;
602 }
603 
604 /// Count duplicated instructions and move the iterators to show where they
605 /// are.
606 /// @param TIB True Iterator Begin
607 /// @param FIB False Iterator Begin
608 /// These two iterators initially point to the first instruction of the two
609 /// blocks, and finally point to the first non-shared instruction.
610 /// @param TIE True Iterator End
611 /// @param FIE False Iterator End
612 /// These two iterators initially point to End() for the two blocks() and
613 /// finally point to the first shared instruction in the tail.
614 /// Upon return [TIB, TIE), and [FIB, FIE) mark the un-duplicated portions of
615 /// two blocks.
616 /// @param Dups1 count of duplicated instructions at the beginning of the 2
617 /// blocks.
618 /// @param Dups2 count of duplicated instructions at the end of the 2 blocks.
619 /// @param SkipUnconditionalBranches if true, Don't make sure that
620 /// unconditional branches at the end of the blocks are the same. True is
621 /// passed when the blocks are analyzable to allow for fallthrough to be
622 /// handled.
623 /// @return false if the shared portion prevents if conversion.
624 bool IfConverter::CountDuplicatedInstructions(
629  unsigned &Dups1, unsigned &Dups2,
631  bool SkipUnconditionalBranches) const {
632 
633  while (TIB != TIE && FIB != FIE) {
634  // Skip dbg_value instructions. These do not count.
635  TIB = skipDebugInstructionsForward(TIB, TIE);
636  if(TIB == TIE)
637  break;
638  FIB = skipDebugInstructionsForward(FIB, FIE);
639  if(FIB == FIE)
640  break;
641  if (!TIB->isIdenticalTo(*FIB))
642  break;
643  // A pred-clobbering instruction in the shared portion prevents
644  // if-conversion.
645  std::vector<MachineOperand> PredDefs;
646  if (TII->DefinesPredicate(*TIB, PredDefs))
647  return false;
648  // If we get all the way to the branch instructions, don't count them.
649  if (!TIB->isBranch())
650  ++Dups1;
651  ++TIB;
652  ++FIB;
653  }
654 
655  // Check for already containing all of the block.
656  if (TIB == TIE || FIB == FIE)
657  return true;
658  // Now, in preparation for counting duplicate instructions at the ends of the
659  // blocks, move the end iterators up past any branch instructions.
660  --TIE;
661  --FIE;
662 
663  // After this point TIB and TIE define an inclusive range, which means that
664  // TIB == TIE is true when there is one more instruction to consider, not at
665  // the end. Because we may not be able to go before TIB, we need a flag to
666  // indicate a completely empty range.
667  bool TEmpty = false, FEmpty = false;
668 
669  // Upon exit TIE and FIE will both point at the last non-shared instruction.
670  // They need to be moved forward to point past the last non-shared
671  // instruction if the range they delimit is non-empty.
672  auto IncrementEndIteratorsOnExit = make_scope_exit([&]() {
673  if (!TEmpty)
674  ++TIE;
675  if (!FEmpty)
676  ++FIE;
677  });
678 
679  if (!TBB.succ_empty() || !FBB.succ_empty()) {
680  if (SkipUnconditionalBranches) {
681  while (!TEmpty && TIE->isUnconditionalBranch())
682  shrinkInclusiveRange(TIB, TIE, TEmpty);
683  while (!FEmpty && FIE->isUnconditionalBranch())
684  shrinkInclusiveRange(FIB, FIE, FEmpty);
685  }
686  }
687 
688  // If Dups1 includes all of a block, then don't count duplicate
689  // instructions at the end of the blocks.
690  if (TEmpty || FEmpty)
691  return true;
692 
693  // Count duplicate instructions at the ends of the blocks.
694  while (!TEmpty && !FEmpty) {
695  // Skip dbg_value instructions. These do not count.
696  TIE = skipDebugInstructionsBackward(TIE, TIB);
697  FIE = skipDebugInstructionsBackward(FIE, FIB);
698  TEmpty = TIE == TIB && TIE->isDebugValue();
699  FEmpty = FIE == FIB && FIE->isDebugValue();
700  if (TEmpty || FEmpty)
701  break;
702  if (!TIE->isIdenticalTo(*FIE))
703  break;
704  // We have to verify that any branch instructions are the same, and then we
705  // don't count them toward the # of duplicate instructions.
706  if (!TIE->isBranch())
707  ++Dups2;
708  shrinkInclusiveRange(TIB, TIE, TEmpty);
709  shrinkInclusiveRange(FIB, FIE, FEmpty);
710  }
711  return true;
712 }
713 
714 /// RescanInstructions - Run ScanInstructions on a pair of blocks.
715 /// @param TIB - True Iterator Begin, points to first non-shared instruction
716 /// @param FIB - False Iterator Begin, points to first non-shared instruction
717 /// @param TIE - True Iterator End, points past last non-shared instruction
718 /// @param FIE - False Iterator End, points past last non-shared instruction
719 /// @param TrueBBI - BBInfo to update for the true block.
720 /// @param FalseBBI - BBInfo to update for the false block.
721 /// @returns - false if either block cannot be predicated or if both blocks end
722 /// with a predicate-clobbering instruction.
723 bool IfConverter::RescanInstructions(
726  BBInfo &TrueBBI, BBInfo &FalseBBI) const {
727  bool BranchUnpredicable = true;
728  TrueBBI.IsUnpredicable = FalseBBI.IsUnpredicable = false;
729  ScanInstructions(TrueBBI, TIB, TIE, BranchUnpredicable);
730  if (TrueBBI.IsUnpredicable)
731  return false;
732  ScanInstructions(FalseBBI, FIB, FIE, BranchUnpredicable);
733  if (FalseBBI.IsUnpredicable)
734  return false;
735  if (TrueBBI.ClobbersPred && FalseBBI.ClobbersPred)
736  return false;
737  return true;
738 }
739 
740 #ifndef NDEBUG
742  MachineBasicBlock *MBB1,
743  MachineBasicBlock *MBB2) {
744  MachineBasicBlock::iterator B1 = MBB1->begin();
745  MachineBasicBlock::iterator B2 = MBB2->begin();
746  MachineBasicBlock::iterator E1 = std::prev(MBB1->end());
747  MachineBasicBlock::iterator E2 = std::prev(MBB2->end());
748  bool Empty1 = false, Empty2 = false;
749  while (!Empty1 && !Empty2) {
750  E1 = skipDebugInstructionsBackward(E1, B1);
751  E2 = skipDebugInstructionsBackward(E2, B2);
752  Empty1 = E1 == B1 && E1->isDebugValue();
753  Empty2 = E2 == B2 && E2->isDebugValue();
754 
755  if (Empty1 && Empty2)
756  break;
757 
758  if (Empty1) {
759  assert(!E2->isBranch() && "Branch mis-match, one block is empty.");
760  break;
761  }
762  if (Empty2) {
763  assert(!E1->isBranch() && "Branch mis-match, one block is empty.");
764  break;
765  }
766 
767  if (E1->isBranch() || E2->isBranch())
768  assert(E1->isIdenticalTo(*E2) &&
769  "Branch mis-match, branch instructions don't match.");
770  else
771  break;
772  shrinkInclusiveRange(B1, E1, Empty1);
773  shrinkInclusiveRange(B2, E2, Empty2);
774  }
775 }
776 #endif
777 
778 /// ValidForkedDiamond - Returns true if the 'true' and 'false' blocks (along
779 /// with their common predecessor) form a diamond if a common tail block is
780 /// extracted.
781 /// While not strictly a diamond, this pattern would form a diamond if
782 /// tail-merging had merged the shared tails.
783 /// EBB
784 /// _/ \_
785 /// | |
786 /// TBB FBB
787 /// / \ / \
788 /// FalseBB TrueBB FalseBB
789 /// Currently only handles analyzable branches.
790 /// Specifically excludes actual diamonds to avoid overlap.
791 bool IfConverter::ValidForkedDiamond(
792  BBInfo &TrueBBI, BBInfo &FalseBBI,
793  unsigned &Dups1, unsigned &Dups2,
794  BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const {
795  Dups1 = Dups2 = 0;
796  if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone ||
797  FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone)
798  return false;
799 
800  if (!TrueBBI.IsBrAnalyzable || !FalseBBI.IsBrAnalyzable)
801  return false;
802  // Don't IfConvert blocks that can't be folded into their predecessor.
803  if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1)
804  return false;
805 
806  // This function is specifically looking for conditional tails, as
807  // unconditional tails are already handled by the standard diamond case.
808  if (TrueBBI.BrCond.size() == 0 ||
809  FalseBBI.BrCond.size() == 0)
810  return false;
811 
812  MachineBasicBlock *TT = TrueBBI.TrueBB;
813  MachineBasicBlock *TF = TrueBBI.FalseBB;
814  MachineBasicBlock *FT = FalseBBI.TrueBB;
815  MachineBasicBlock *FF = FalseBBI.FalseBB;
816 
817  if (!TT)
818  TT = getNextBlock(*TrueBBI.BB);
819  if (!TF)
820  TF = getNextBlock(*TrueBBI.BB);
821  if (!FT)
822  FT = getNextBlock(*FalseBBI.BB);
823  if (!FF)
824  FF = getNextBlock(*FalseBBI.BB);
825 
826  if (!TT || !TF)
827  return false;
828 
829  // Check successors. If they don't match, bail.
830  if (!((TT == FT && TF == FF) || (TF == FT && TT == FF)))
831  return false;
832 
833  bool FalseReversed = false;
834  if (TF == FT && TT == FF) {
835  // If the branches are opposing, but we can't reverse, don't do it.
836  if (!FalseBBI.IsBrReversible)
837  return false;
838  FalseReversed = true;
839  reverseBranchCondition(FalseBBI);
840  }
841  auto UnReverseOnExit = make_scope_exit([&]() {
842  if (FalseReversed)
843  reverseBranchCondition(FalseBBI);
844  });
845 
846  // Count duplicate instructions at the beginning of the true and false blocks.
847  MachineBasicBlock::iterator TIB = TrueBBI.BB->begin();
848  MachineBasicBlock::iterator FIB = FalseBBI.BB->begin();
849  MachineBasicBlock::iterator TIE = TrueBBI.BB->end();
850  MachineBasicBlock::iterator FIE = FalseBBI.BB->end();
851  if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2,
852  *TrueBBI.BB, *FalseBBI.BB,
853  /* SkipUnconditionalBranches */ true))
854  return false;
855 
856  TrueBBICalc.BB = TrueBBI.BB;
857  FalseBBICalc.BB = FalseBBI.BB;
858  if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc))
859  return false;
860 
861  // The size is used to decide whether to if-convert, and the shared portions
862  // are subtracted off. Because of the subtraction, we just use the size that
863  // was calculated by the original ScanInstructions, as it is correct.
864  TrueBBICalc.NonPredSize = TrueBBI.NonPredSize;
865  FalseBBICalc.NonPredSize = FalseBBI.NonPredSize;
866  return true;
867 }
868 
869 /// ValidDiamond - Returns true if the 'true' and 'false' blocks (along
870 /// with their common predecessor) forms a valid diamond shape for ifcvt.
871 bool IfConverter::ValidDiamond(
872  BBInfo &TrueBBI, BBInfo &FalseBBI,
873  unsigned &Dups1, unsigned &Dups2,
874  BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const {
875  Dups1 = Dups2 = 0;
876  if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone ||
877  FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone)
878  return false;
879 
880  MachineBasicBlock *TT = TrueBBI.TrueBB;
881  MachineBasicBlock *FT = FalseBBI.TrueBB;
882 
883  if (!TT && blockAlwaysFallThrough(TrueBBI))
884  TT = getNextBlock(*TrueBBI.BB);
885  if (!FT && blockAlwaysFallThrough(FalseBBI))
886  FT = getNextBlock(*FalseBBI.BB);
887  if (TT != FT)
888  return false;
889  if (!TT && (TrueBBI.IsBrAnalyzable || FalseBBI.IsBrAnalyzable))
890  return false;
891  if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1)
892  return false;
893 
894  // FIXME: Allow true block to have an early exit?
895  if (TrueBBI.FalseBB || FalseBBI.FalseBB)
896  return false;
897 
898  // Count duplicate instructions at the beginning and end of the true and
899  // false blocks.
900  // Skip unconditional branches only if we are considering an analyzable
901  // diamond. Otherwise the branches must be the same.
902  bool SkipUnconditionalBranches =
903  TrueBBI.IsBrAnalyzable && FalseBBI.IsBrAnalyzable;
904  MachineBasicBlock::iterator TIB = TrueBBI.BB->begin();
905  MachineBasicBlock::iterator FIB = FalseBBI.BB->begin();
906  MachineBasicBlock::iterator TIE = TrueBBI.BB->end();
907  MachineBasicBlock::iterator FIE = FalseBBI.BB->end();
908  if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2,
909  *TrueBBI.BB, *FalseBBI.BB,
910  SkipUnconditionalBranches))
911  return false;
912 
913  TrueBBICalc.BB = TrueBBI.BB;
914  FalseBBICalc.BB = FalseBBI.BB;
915  if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc))
916  return false;
917  // The size is used to decide whether to if-convert, and the shared portions
918  // are subtracted off. Because of the subtraction, we just use the size that
919  // was calculated by the original ScanInstructions, as it is correct.
920  TrueBBICalc.NonPredSize = TrueBBI.NonPredSize;
921  FalseBBICalc.NonPredSize = FalseBBI.NonPredSize;
922  return true;
923 }
924 
925 /// AnalyzeBranches - Look at the branches at the end of a block to determine if
926 /// the block is predicable.
927 void IfConverter::AnalyzeBranches(BBInfo &BBI) {
928  if (BBI.IsDone)
929  return;
930 
931  BBI.TrueBB = BBI.FalseBB = nullptr;
932  BBI.BrCond.clear();
933  BBI.IsBrAnalyzable =
934  !TII->analyzeBranch(*BBI.BB, BBI.TrueBB, BBI.FalseBB, BBI.BrCond);
935  SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
936  BBI.IsBrReversible = (RevCond.size() == 0) ||
937  !TII->reverseBranchCondition(RevCond);
938  BBI.HasFallThrough = BBI.IsBrAnalyzable && BBI.FalseBB == nullptr;
939 
940  if (BBI.BrCond.size()) {
941  // No false branch. This BB must end with a conditional branch and a
942  // fallthrough.
943  if (!BBI.FalseBB)
944  BBI.FalseBB = findFalseBlock(BBI.BB, BBI.TrueBB);
945  if (!BBI.FalseBB) {
946  // Malformed bcc? True and false blocks are the same?
947  BBI.IsUnpredicable = true;
948  }
949  }
950 }
951 
952 /// ScanInstructions - Scan all the instructions in the block to determine if
953 /// the block is predicable. In most cases, that means all the instructions
954 /// in the block are isPredicable(). Also checks if the block contains any
955 /// instruction which can clobber a predicate (e.g. condition code register).
956 /// If so, the block is not predicable unless it's the last instruction.
957 void IfConverter::ScanInstructions(BBInfo &BBI,
960  bool BranchUnpredicable) const {
961  if (BBI.IsDone || BBI.IsUnpredicable)
962  return;
963 
964  bool AlreadyPredicated = !BBI.Predicate.empty();
965 
966  BBI.NonPredSize = 0;
967  BBI.ExtraCost = 0;
968  BBI.ExtraCost2 = 0;
969  BBI.ClobbersPred = false;
970  for (MachineInstr &MI : make_range(Begin, End)) {
971  if (MI.isDebugValue())
972  continue;
973 
974  // It's unsafe to duplicate convergent instructions in this context, so set
975  // BBI.CannotBeCopied to true if MI is convergent. To see why, consider the
976  // following CFG, which is subject to our "simple" transformation.
977  //
978  // BB0 // if (c1) goto BB1; else goto BB2;
979  // / \
980  // BB1 |
981  // | BB2 // if (c2) goto TBB; else goto FBB;
982  // | / |
983  // | / |
984  // TBB |
985  // | |
986  // | FBB
987  // |
988  // exit
989  //
990  // Suppose we want to move TBB's contents up into BB1 and BB2 (in BB1 they'd
991  // be unconditional, and in BB2, they'd be predicated upon c2), and suppose
992  // TBB contains a convergent instruction. This is safe iff doing so does
993  // not add a control-flow dependency to the convergent instruction -- i.e.,
994  // it's safe iff the set of control flows that leads us to the convergent
995  // instruction does not get smaller after the transformation.
996  //
997  // Originally we executed TBB if c1 || c2. After the transformation, there
998  // are two copies of TBB's instructions. We get to the first if c1, and we
999  // get to the second if !c1 && c2.
1000  //
1001  // There are clearly fewer ways to satisfy the condition "c1" than
1002  // "c1 || c2". Since we've shrunk the set of control flows which lead to
1003  // our convergent instruction, the transformation is unsafe.
1004  if (MI.isNotDuplicable() || MI.isConvergent())
1005  BBI.CannotBeCopied = true;
1006 
1007  bool isPredicated = TII->isPredicated(MI);
1008  bool isCondBr = BBI.IsBrAnalyzable && MI.isConditionalBranch();
1009 
1010  if (BranchUnpredicable && MI.isBranch()) {
1011  BBI.IsUnpredicable = true;
1012  return;
1013  }
1014 
1015  // A conditional branch is not predicable, but it may be eliminated.
1016  if (isCondBr)
1017  continue;
1018 
1019  if (!isPredicated) {
1020  BBI.NonPredSize++;
1021  unsigned ExtraPredCost = TII->getPredicationCost(MI);
1022  unsigned NumCycles = SchedModel.computeInstrLatency(&MI, false);
1023  if (NumCycles > 1)
1024  BBI.ExtraCost += NumCycles-1;
1025  BBI.ExtraCost2 += ExtraPredCost;
1026  } else if (!AlreadyPredicated) {
1027  // FIXME: This instruction is already predicated before the
1028  // if-conversion pass. It's probably something like a conditional move.
1029  // Mark this block unpredicable for now.
1030  BBI.IsUnpredicable = true;
1031  return;
1032  }
1033 
1034  if (BBI.ClobbersPred && !isPredicated) {
1035  // Predicate modification instruction should end the block (except for
1036  // already predicated instructions and end of block branches).
1037  // Predicate may have been modified, the subsequent (currently)
1038  // unpredicated instructions cannot be correctly predicated.
1039  BBI.IsUnpredicable = true;
1040  return;
1041  }
1042 
1043  // FIXME: Make use of PredDefs? e.g. ADDC, SUBC sets predicates but are
1044  // still potentially predicable.
1045  std::vector<MachineOperand> PredDefs;
1046  if (TII->DefinesPredicate(MI, PredDefs))
1047  BBI.ClobbersPred = true;
1048 
1049  if (!TII->isPredicable(MI)) {
1050  BBI.IsUnpredicable = true;
1051  return;
1052  }
1053  }
1054 }
1055 
1056 /// Determine if the block is a suitable candidate to be predicated by the
1057 /// specified predicate.
1058 /// @param BBI BBInfo for the block to check
1059 /// @param Pred Predicate array for the branch that leads to BBI
1060 /// @param isTriangle true if the Analysis is for a triangle
1061 /// @param RevBranch true if Reverse(Pred) leads to BBI (e.g. BBI is the false
1062 /// case
1063 /// @param hasCommonTail true if BBI shares a tail with a sibling block that
1064 /// contains any instruction that would make the block unpredicable.
1065 bool IfConverter::FeasibilityAnalysis(BBInfo &BBI,
1067  bool isTriangle, bool RevBranch,
1068  bool hasCommonTail) {
1069  // If the block is dead or unpredicable, then it cannot be predicated.
1070  // Two blocks may share a common unpredicable tail, but this doesn't prevent
1071  // them from being if-converted. The non-shared portion is assumed to have
1072  // been checked
1073  if (BBI.IsDone || (BBI.IsUnpredicable && !hasCommonTail))
1074  return false;
1075 
1076  // If it is already predicated but we couldn't analyze its terminator, the
1077  // latter might fallthrough, but we can't determine where to.
1078  // Conservatively avoid if-converting again.
1079  if (BBI.Predicate.size() && !BBI.IsBrAnalyzable)
1080  return false;
1081 
1082  // If it is already predicated, check if the new predicate subsumes
1083  // its predicate.
1084  if (BBI.Predicate.size() && !TII->SubsumesPredicate(Pred, BBI.Predicate))
1085  return false;
1086 
1087  if (!hasCommonTail && BBI.BrCond.size()) {
1088  if (!isTriangle)
1089  return false;
1090 
1091  // Test predicate subsumption.
1092  SmallVector<MachineOperand, 4> RevPred(Pred.begin(), Pred.end());
1093  SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end());
1094  if (RevBranch) {
1095  if (TII->reverseBranchCondition(Cond))
1096  return false;
1097  }
1098  if (TII->reverseBranchCondition(RevPred) ||
1099  !TII->SubsumesPredicate(Cond, RevPred))
1100  return false;
1101  }
1102 
1103  return true;
1104 }
1105 
1106 /// Analyze the structure of the sub-CFG starting from the specified block.
1107 /// Record its successors and whether it looks like an if-conversion candidate.
1108 void IfConverter::AnalyzeBlock(
1109  MachineBasicBlock &MBB, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) {
1110  struct BBState {
1111  BBState(MachineBasicBlock &MBB) : MBB(&MBB), SuccsAnalyzed(false) {}
1113 
1114  /// This flag is true if MBB's successors have been analyzed.
1115  bool SuccsAnalyzed;
1116  };
1117 
1118  // Push MBB to the stack.
1119  SmallVector<BBState, 16> BBStack(1, MBB);
1120 
1121  while (!BBStack.empty()) {
1122  BBState &State = BBStack.back();
1123  MachineBasicBlock *BB = State.MBB;
1124  BBInfo &BBI = BBAnalysis[BB->getNumber()];
1125 
1126  if (!State.SuccsAnalyzed) {
1127  if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed) {
1128  BBStack.pop_back();
1129  continue;
1130  }
1131 
1132  BBI.BB = BB;
1133  BBI.IsBeingAnalyzed = true;
1134 
1135  AnalyzeBranches(BBI);
1136  MachineBasicBlock::iterator Begin = BBI.BB->begin();
1137  MachineBasicBlock::iterator End = BBI.BB->end();
1138  ScanInstructions(BBI, Begin, End);
1139 
1140  // Unanalyzable or ends with fallthrough or unconditional branch, or if is
1141  // not considered for ifcvt anymore.
1142  if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) {
1143  BBI.IsBeingAnalyzed = false;
1144  BBI.IsAnalyzed = true;
1145  BBStack.pop_back();
1146  continue;
1147  }
1148 
1149  // Do not ifcvt if either path is a back edge to the entry block.
1150  if (BBI.TrueBB == BB || BBI.FalseBB == BB) {
1151  BBI.IsBeingAnalyzed = false;
1152  BBI.IsAnalyzed = true;
1153  BBStack.pop_back();
1154  continue;
1155  }
1156 
1157  // Do not ifcvt if true and false fallthrough blocks are the same.
1158  if (!BBI.FalseBB) {
1159  BBI.IsBeingAnalyzed = false;
1160  BBI.IsAnalyzed = true;
1161  BBStack.pop_back();
1162  continue;
1163  }
1164 
1165  // Push the False and True blocks to the stack.
1166  State.SuccsAnalyzed = true;
1167  BBStack.push_back(*BBI.FalseBB);
1168  BBStack.push_back(*BBI.TrueBB);
1169  continue;
1170  }
1171 
1172  BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1173  BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1174 
1175  if (TrueBBI.IsDone && FalseBBI.IsDone) {
1176  BBI.IsBeingAnalyzed = false;
1177  BBI.IsAnalyzed = true;
1178  BBStack.pop_back();
1179  continue;
1180  }
1181 
1183  RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
1184  bool CanRevCond = !TII->reverseBranchCondition(RevCond);
1185 
1186  unsigned Dups = 0;
1187  unsigned Dups2 = 0;
1188  bool TNeedSub = !TrueBBI.Predicate.empty();
1189  bool FNeedSub = !FalseBBI.Predicate.empty();
1190  bool Enqueued = false;
1191 
1192  BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB);
1193 
1194  if (CanRevCond) {
1195  BBInfo TrueBBICalc, FalseBBICalc;
1196  auto feasibleDiamond = [&]() {
1197  bool MeetsSize = MeetIfcvtSizeLimit(
1198  *TrueBBI.BB, (TrueBBICalc.NonPredSize - (Dups + Dups2) +
1199  TrueBBICalc.ExtraCost), TrueBBICalc.ExtraCost2,
1200  *FalseBBI.BB, (FalseBBICalc.NonPredSize - (Dups + Dups2) +
1201  FalseBBICalc.ExtraCost), FalseBBICalc.ExtraCost2,
1202  Prediction);
1203  bool TrueFeasible = FeasibilityAnalysis(TrueBBI, BBI.BrCond,
1204  /* IsTriangle */ false, /* RevCond */ false,
1205  /* hasCommonTail */ true);
1206  bool FalseFeasible = FeasibilityAnalysis(FalseBBI, RevCond,
1207  /* IsTriangle */ false, /* RevCond */ false,
1208  /* hasCommonTail */ true);
1209  return MeetsSize && TrueFeasible && FalseFeasible;
1210  };
1211 
1212  if (ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2,
1213  TrueBBICalc, FalseBBICalc)) {
1214  if (feasibleDiamond()) {
1215  // Diamond:
1216  // EBB
1217  // / \_
1218  // | |
1219  // TBB FBB
1220  // \ /
1221  // TailBB
1222  // Note TailBB can be empty.
1223  Tokens.push_back(llvm::make_unique<IfcvtToken>(
1224  BBI, ICDiamond, TNeedSub | FNeedSub, Dups, Dups2,
1225  (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred));
1226  Enqueued = true;
1227  }
1228  } else if (ValidForkedDiamond(TrueBBI, FalseBBI, Dups, Dups2,
1229  TrueBBICalc, FalseBBICalc)) {
1230  if (feasibleDiamond()) {
1231  // ForkedDiamond:
1232  // if TBB and FBB have a common tail that includes their conditional
1233  // branch instructions, then we can If Convert this pattern.
1234  // EBB
1235  // _/ \_
1236  // | |
1237  // TBB FBB
1238  // / \ / \
1239  // FalseBB TrueBB FalseBB
1240  //
1241  Tokens.push_back(llvm::make_unique<IfcvtToken>(
1242  BBI, ICForkedDiamond, TNeedSub | FNeedSub, Dups, Dups2,
1243  (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred));
1244  Enqueued = true;
1245  }
1246  }
1247  }
1248 
1249  if (ValidTriangle(TrueBBI, FalseBBI, false, Dups, Prediction) &&
1250  MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
1251  TrueBBI.ExtraCost2, Prediction) &&
1252  FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) {
1253  // Triangle:
1254  // EBB
1255  // | \_
1256  // | |
1257  // | TBB
1258  // | /
1259  // FBB
1260  Tokens.push_back(
1261  llvm::make_unique<IfcvtToken>(BBI, ICTriangle, TNeedSub, Dups));
1262  Enqueued = true;
1263  }
1264 
1265  if (ValidTriangle(TrueBBI, FalseBBI, true, Dups, Prediction) &&
1266  MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
1267  TrueBBI.ExtraCost2, Prediction) &&
1268  FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) {
1269  Tokens.push_back(
1270  llvm::make_unique<IfcvtToken>(BBI, ICTriangleRev, TNeedSub, Dups));
1271  Enqueued = true;
1272  }
1273 
1274  if (ValidSimple(TrueBBI, Dups, Prediction) &&
1275  MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
1276  TrueBBI.ExtraCost2, Prediction) &&
1277  FeasibilityAnalysis(TrueBBI, BBI.BrCond)) {
1278  // Simple (split, no rejoin):
1279  // EBB
1280  // | \_
1281  // | |
1282  // | TBB---> exit
1283  // |
1284  // FBB
1285  Tokens.push_back(
1286  llvm::make_unique<IfcvtToken>(BBI, ICSimple, TNeedSub, Dups));
1287  Enqueued = true;
1288  }
1289 
1290  if (CanRevCond) {
1291  // Try the other path...
1292  if (ValidTriangle(FalseBBI, TrueBBI, false, Dups,
1293  Prediction.getCompl()) &&
1294  MeetIfcvtSizeLimit(*FalseBBI.BB,
1295  FalseBBI.NonPredSize + FalseBBI.ExtraCost,
1296  FalseBBI.ExtraCost2, Prediction.getCompl()) &&
1297  FeasibilityAnalysis(FalseBBI, RevCond, true)) {
1298  Tokens.push_back(llvm::make_unique<IfcvtToken>(BBI, ICTriangleFalse,
1299  FNeedSub, Dups));
1300  Enqueued = true;
1301  }
1302 
1303  if (ValidTriangle(FalseBBI, TrueBBI, true, Dups,
1304  Prediction.getCompl()) &&
1305  MeetIfcvtSizeLimit(*FalseBBI.BB,
1306  FalseBBI.NonPredSize + FalseBBI.ExtraCost,
1307  FalseBBI.ExtraCost2, Prediction.getCompl()) &&
1308  FeasibilityAnalysis(FalseBBI, RevCond, true, true)) {
1309  Tokens.push_back(
1310  llvm::make_unique<IfcvtToken>(BBI, ICTriangleFRev, FNeedSub, Dups));
1311  Enqueued = true;
1312  }
1313 
1314  if (ValidSimple(FalseBBI, Dups, Prediction.getCompl()) &&
1315  MeetIfcvtSizeLimit(*FalseBBI.BB,
1316  FalseBBI.NonPredSize + FalseBBI.ExtraCost,
1317  FalseBBI.ExtraCost2, Prediction.getCompl()) &&
1318  FeasibilityAnalysis(FalseBBI, RevCond)) {
1319  Tokens.push_back(
1320  llvm::make_unique<IfcvtToken>(BBI, ICSimpleFalse, FNeedSub, Dups));
1321  Enqueued = true;
1322  }
1323  }
1324 
1325  BBI.IsEnqueued = Enqueued;
1326  BBI.IsBeingAnalyzed = false;
1327  BBI.IsAnalyzed = true;
1328  BBStack.pop_back();
1329  }
1330 }
1331 
1332 /// Analyze all blocks and find entries for all if-conversion candidates.
1333 void IfConverter::AnalyzeBlocks(
1334  MachineFunction &MF, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) {
1335  for (MachineBasicBlock &MBB : MF)
1336  AnalyzeBlock(MBB, Tokens);
1337 
1338  // Sort to favor more complex ifcvt scheme.
1339  std::stable_sort(Tokens.begin(), Tokens.end(), IfcvtTokenCmp);
1340 }
1341 
1342 /// Returns true either if ToMBB is the next block after MBB or that all the
1343 /// intervening blocks are empty (given MBB can fall through to its next block).
1346  MachineFunction::iterator I = std::next(PI);
1349  while (I != TI) {
1350  // Check isSuccessor to avoid case where the next block is empty, but
1351  // it's not a successor.
1352  if (I == E || !I->empty() || !PI->isSuccessor(&*I))
1353  return false;
1354  PI = I++;
1355  }
1356  return true;
1357 }
1358 
1359 /// Invalidate predecessor BB info so it would be re-analyzed to determine if it
1360 /// can be if-converted. If predecessor is already enqueued, dequeue it!
1361 void IfConverter::InvalidatePreds(MachineBasicBlock &MBB) {
1362  for (const MachineBasicBlock *Predecessor : MBB.predecessors()) {
1363  BBInfo &PBBI = BBAnalysis[Predecessor->getNumber()];
1364  if (PBBI.IsDone || PBBI.BB == &MBB)
1365  continue;
1366  PBBI.IsAnalyzed = false;
1367  PBBI.IsEnqueued = false;
1368  }
1369 }
1370 
1371 /// Inserts an unconditional branch from \p MBB to \p ToMBB.
1373  const TargetInstrInfo *TII) {
1374  DebugLoc dl; // FIXME: this is nowhere
1376  TII->insertBranch(MBB, &ToMBB, nullptr, NoCond, dl);
1377 }
1378 
1379 /// Remove true / false edges if either / both are no longer successors.
1380 void IfConverter::RemoveExtraEdges(BBInfo &BBI) {
1381  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
1383  if (!TII->analyzeBranch(*BBI.BB, TBB, FBB, Cond))
1384  BBI.BB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty());
1385 }
1386 
1387 /// Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all
1388 /// values defined in MI which are also live/used by MI.
1390  const TargetRegisterInfo *TRI = MI.getParent()->getParent()
1392 
1393  // Before stepping forward past MI, remember which regs were live
1394  // before MI. This is needed to set the Undef flag only when reg is
1395  // dead.
1396  SparseSet<unsigned> LiveBeforeMI;
1397  LiveBeforeMI.setUniverse(TRI->getNumRegs());
1398  for (unsigned Reg : Redefs)
1399  LiveBeforeMI.insert(Reg);
1400 
1402  Redefs.stepForward(MI, Clobbers);
1403 
1404  // Now add the implicit uses for each of the clobbered values.
1405  for (auto Clobber : Clobbers) {
1406  // FIXME: Const cast here is nasty, but better than making StepForward
1407  // take a mutable instruction instead of const.
1408  unsigned Reg = Clobber.first;
1409  MachineOperand &Op = const_cast<MachineOperand&>(*Clobber.second);
1410  MachineInstr *OpMI = Op.getParent();
1411  MachineInstrBuilder MIB(*OpMI->getParent()->getParent(), OpMI);
1412  if (Op.isRegMask()) {
1413  // First handle regmasks. They clobber any entries in the mask which
1414  // means that we need a def for those registers.
1415  if (LiveBeforeMI.count(Reg))
1416  MIB.addReg(Reg, RegState::Implicit);
1417 
1418  // We also need to add an implicit def of this register for the later
1419  // use to read from.
1420  // For the register allocator to have allocated a register clobbered
1421  // by the call which is used later, it must be the case that
1422  // the call doesn't return.
1424  continue;
1425  }
1426  assert(Op.isReg() && "Register operand required");
1427  if (Op.isDead()) {
1428  // If we found a dead def, but it needs to be live, then remove the dead
1429  // flag.
1430  if (Redefs.contains(Op.getReg()))
1431  Op.setIsDead(false);
1432  }
1433  if (LiveBeforeMI.count(Reg))
1434  MIB.addReg(Reg, RegState::Implicit);
1435  else {
1436  bool HasLiveSubReg = false;
1437  for (MCSubRegIterator S(Reg, TRI); S.isValid(); ++S) {
1438  if (!LiveBeforeMI.count(*S))
1439  continue;
1440  HasLiveSubReg = true;
1441  break;
1442  }
1443  if (HasLiveSubReg)
1444  MIB.addReg(Reg, RegState::Implicit);
1445  }
1446  }
1447 }
1448 
1449 /// Remove kill flags from operands with a registers in the \p DontKill set.
1450 static void RemoveKills(MachineInstr &MI, const LivePhysRegs &DontKill) {
1451  for (MIBundleOperands O(MI); O.isValid(); ++O) {
1452  if (!O->isReg() || !O->isKill())
1453  continue;
1454  if (DontKill.contains(O->getReg()))
1455  O->setIsKill(false);
1456  }
1457 }
1458 
1459 /// Walks a range of machine instructions and removes kill flags for registers
1460 /// in the \p DontKill set.
1463  const LivePhysRegs &DontKill,
1464  const MCRegisterInfo &MCRI) {
1465  for (MachineInstr &MI : make_range(I, E))
1466  RemoveKills(MI, DontKill);
1467 }
1468 
1469 /// If convert a simple (split, no rejoin) sub-CFG.
1470 bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) {
1471  BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1472  BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1473  BBInfo *CvtBBI = &TrueBBI;
1474  BBInfo *NextBBI = &FalseBBI;
1475 
1476  SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end());
1477  if (Kind == ICSimpleFalse)
1478  std::swap(CvtBBI, NextBBI);
1479 
1480  MachineBasicBlock &CvtMBB = *CvtBBI->BB;
1481  MachineBasicBlock &NextMBB = *NextBBI->BB;
1482  if (CvtBBI->IsDone ||
1483  (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) {
1484  // Something has changed. It's no longer safe to predicate this block.
1485  BBI.IsAnalyzed = false;
1486  CvtBBI->IsAnalyzed = false;
1487  return false;
1488  }
1489 
1490  if (CvtMBB.hasAddressTaken())
1491  // Conservatively abort if-conversion if BB's address is taken.
1492  return false;
1493 
1494  if (Kind == ICSimpleFalse)
1495  if (TII->reverseBranchCondition(Cond))
1496  llvm_unreachable("Unable to reverse branch condition!");
1497 
1498  Redefs.init(*TRI);
1499  DontKill.init(*TRI);
1500 
1501  if (MRI->tracksLiveness()) {
1502  // Initialize liveins to the first BB. These are potentiall redefined by
1503  // predicated instructions.
1504  Redefs.addLiveIns(CvtMBB);
1505  Redefs.addLiveIns(NextMBB);
1506  // Compute a set of registers which must not be killed by instructions in
1507  // BB1: This is everything live-in to BB2.
1508  DontKill.addLiveIns(NextMBB);
1509  }
1510 
1511  if (CvtMBB.pred_size() > 1) {
1512  BBI.NonPredSize -= TII->removeBranch(*BBI.BB);
1513  // Copy instructions in the true block, predicate them, and add them to
1514  // the entry block.
1515  CopyAndPredicateBlock(BBI, *CvtBBI, Cond);
1516 
1517  // RemoveExtraEdges won't work if the block has an unanalyzable branch, so
1518  // explicitly remove CvtBBI as a successor.
1519  BBI.BB->removeSuccessor(&CvtMBB, true);
1520  } else {
1521  RemoveKills(CvtMBB.begin(), CvtMBB.end(), DontKill, *TRI);
1522  PredicateBlock(*CvtBBI, CvtMBB.end(), Cond);
1523 
1524  // Merge converted block into entry block.
1525  BBI.NonPredSize -= TII->removeBranch(*BBI.BB);
1526  MergeBlocks(BBI, *CvtBBI);
1527  }
1528 
1529  bool IterIfcvt = true;
1530  if (!canFallThroughTo(*BBI.BB, NextMBB)) {
1531  InsertUncondBranch(*BBI.BB, NextMBB, TII);
1532  BBI.HasFallThrough = false;
1533  // Now ifcvt'd block will look like this:
1534  // BB:
1535  // ...
1536  // t, f = cmp
1537  // if t op
1538  // b BBf
1539  //
1540  // We cannot further ifcvt this block because the unconditional branch
1541  // will have to be predicated on the new condition, that will not be
1542  // available if cmp executes.
1543  IterIfcvt = false;
1544  }
1545 
1546  RemoveExtraEdges(BBI);
1547 
1548  // Update block info. BB can be iteratively if-converted.
1549  if (!IterIfcvt)
1550  BBI.IsDone = true;
1551  InvalidatePreds(*BBI.BB);
1552  CvtBBI->IsDone = true;
1553 
1554  // FIXME: Must maintain LiveIns.
1555  return true;
1556 }
1557 
1558 /// If convert a triangle sub-CFG.
1559 bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
1560  BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1561  BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1562  BBInfo *CvtBBI = &TrueBBI;
1563  BBInfo *NextBBI = &FalseBBI;
1564  DebugLoc dl; // FIXME: this is nowhere
1565 
1566  SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end());
1567  if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
1568  std::swap(CvtBBI, NextBBI);
1569 
1570  MachineBasicBlock &CvtMBB = *CvtBBI->BB;
1571  MachineBasicBlock &NextMBB = *NextBBI->BB;
1572  if (CvtBBI->IsDone ||
1573  (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) {
1574  // Something has changed. It's no longer safe to predicate this block.
1575  BBI.IsAnalyzed = false;
1576  CvtBBI->IsAnalyzed = false;
1577  return false;
1578  }
1579 
1580  if (CvtMBB.hasAddressTaken())
1581  // Conservatively abort if-conversion if BB's address is taken.
1582  return false;
1583 
1584  if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
1585  if (TII->reverseBranchCondition(Cond))
1586  llvm_unreachable("Unable to reverse branch condition!");
1587 
1588  if (Kind == ICTriangleRev || Kind == ICTriangleFRev) {
1589  if (reverseBranchCondition(*CvtBBI)) {
1590  // BB has been changed, modify its predecessors (except for this
1591  // one) so they don't get ifcvt'ed based on bad intel.
1592  for (MachineBasicBlock *PBB : CvtMBB.predecessors()) {
1593  if (PBB == BBI.BB)
1594  continue;
1595  BBInfo &PBBI = BBAnalysis[PBB->getNumber()];
1596  if (PBBI.IsEnqueued) {
1597  PBBI.IsAnalyzed = false;
1598  PBBI.IsEnqueued = false;
1599  }
1600  }
1601  }
1602  }
1603 
1604  // Initialize liveins to the first BB. These are potentially redefined by
1605  // predicated instructions.
1606  Redefs.init(*TRI);
1607  if (MRI->tracksLiveness()) {
1608  Redefs.addLiveIns(CvtMBB);
1609  Redefs.addLiveIns(NextMBB);
1610  }
1611 
1612  DontKill.clear();
1613 
1614  bool HasEarlyExit = CvtBBI->FalseBB != nullptr;
1615  BranchProbability CvtNext, CvtFalse, BBNext, BBCvt;
1616 
1617  if (HasEarlyExit) {
1618  // Get probabilities before modifying CvtMBB and BBI.BB.
1619  CvtNext = MBPI->getEdgeProbability(&CvtMBB, &NextMBB);
1620  CvtFalse = MBPI->getEdgeProbability(&CvtMBB, CvtBBI->FalseBB);
1621  BBNext = MBPI->getEdgeProbability(BBI.BB, &NextMBB);
1622  BBCvt = MBPI->getEdgeProbability(BBI.BB, &CvtMBB);
1623  }
1624 
1625  if (CvtMBB.pred_size() > 1) {
1626  BBI.NonPredSize -= TII->removeBranch(*BBI.BB);
1627  // Copy instructions in the true block, predicate them, and add them to
1628  // the entry block.
1629  CopyAndPredicateBlock(BBI, *CvtBBI, Cond, true);
1630 
1631  // RemoveExtraEdges won't work if the block has an unanalyzable branch, so
1632  // explicitly remove CvtBBI as a successor.
1633  BBI.BB->removeSuccessor(&CvtMBB, true);
1634  } else {
1635  // Predicate the 'true' block after removing its branch.
1636  CvtBBI->NonPredSize -= TII->removeBranch(CvtMBB);
1637  PredicateBlock(*CvtBBI, CvtMBB.end(), Cond);
1638 
1639  // Now merge the entry of the triangle with the true block.
1640  BBI.NonPredSize -= TII->removeBranch(*BBI.BB);
1641  MergeBlocks(BBI, *CvtBBI, false);
1642  }
1643 
1644  // If 'true' block has a 'false' successor, add an exit branch to it.
1645  if (HasEarlyExit) {
1646  SmallVector<MachineOperand, 4> RevCond(CvtBBI->BrCond.begin(),
1647  CvtBBI->BrCond.end());
1648  if (TII->reverseBranchCondition(RevCond))
1649  llvm_unreachable("Unable to reverse branch condition!");
1650 
1651  // Update the edge probability for both CvtBBI->FalseBB and NextBBI.
1652  // NewNext = New_Prob(BBI.BB, NextMBB) =
1653  // Prob(BBI.BB, NextMBB) +
1654  // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, NextMBB)
1655  // NewFalse = New_Prob(BBI.BB, CvtBBI->FalseBB) =
1656  // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, CvtBBI->FalseBB)
1657  auto NewTrueBB = getNextBlock(*BBI.BB);
1658  auto NewNext = BBNext + BBCvt * CvtNext;
1659  auto NewTrueBBIter = find(BBI.BB->successors(), NewTrueBB);
1660  if (NewTrueBBIter != BBI.BB->succ_end())
1661  BBI.BB->setSuccProbability(NewTrueBBIter, NewNext);
1662 
1663  auto NewFalse = BBCvt * CvtFalse;
1664  TII->insertBranch(*BBI.BB, CvtBBI->FalseBB, nullptr, RevCond, dl);
1665  BBI.BB->addSuccessor(CvtBBI->FalseBB, NewFalse);
1666  }
1667 
1668  // Merge in the 'false' block if the 'false' block has no other
1669  // predecessors. Otherwise, add an unconditional branch to 'false'.
1670  bool FalseBBDead = false;
1671  bool IterIfcvt = true;
1672  bool isFallThrough = canFallThroughTo(*BBI.BB, NextMBB);
1673  if (!isFallThrough) {
1674  // Only merge them if the true block does not fallthrough to the false
1675  // block. By not merging them, we make it possible to iteratively
1676  // ifcvt the blocks.
1677  if (!HasEarlyExit &&
1678  NextMBB.pred_size() == 1 && !NextBBI->HasFallThrough &&
1679  !NextMBB.hasAddressTaken()) {
1680  MergeBlocks(BBI, *NextBBI);
1681  FalseBBDead = true;
1682  } else {
1683  InsertUncondBranch(*BBI.BB, NextMBB, TII);
1684  BBI.HasFallThrough = false;
1685  }
1686  // Mixed predicated and unpredicated code. This cannot be iteratively
1687  // predicated.
1688  IterIfcvt = false;
1689  }
1690 
1691  RemoveExtraEdges(BBI);
1692 
1693  // Update block info. BB can be iteratively if-converted.
1694  if (!IterIfcvt)
1695  BBI.IsDone = true;
1696  InvalidatePreds(*BBI.BB);
1697  CvtBBI->IsDone = true;
1698  if (FalseBBDead)
1699  NextBBI->IsDone = true;
1700 
1701  // FIXME: Must maintain LiveIns.
1702  return true;
1703 }
1704 
1705 /// Common code shared between diamond conversions.
1706 /// \p BBI, \p TrueBBI, and \p FalseBBI form the diamond shape.
1707 /// \p NumDups1 - number of shared instructions at the beginning of \p TrueBBI
1708 /// and FalseBBI
1709 /// \p NumDups2 - number of shared instructions at the end of \p TrueBBI
1710 /// and \p FalseBBI
1711 /// \p RemoveBranch - Remove the common branch of the two blocks before
1712 /// predicating. Only false for unanalyzable fallthrough
1713 /// cases. The caller will replace the branch if necessary.
1714 /// \p MergeAddEdges - Add successor edges when merging blocks. Only false for
1715 /// unanalyzable fallthrough
1716 bool IfConverter::IfConvertDiamondCommon(
1717  BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI,
1718  unsigned NumDups1, unsigned NumDups2,
1719  bool TClobbersPred, bool FClobbersPred,
1720  bool RemoveBranch, bool MergeAddEdges) {
1721 
1722  if (TrueBBI.IsDone || FalseBBI.IsDone ||
1723  TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) {
1724  // Something has changed. It's no longer safe to predicate these blocks.
1725  BBI.IsAnalyzed = false;
1726  TrueBBI.IsAnalyzed = false;
1727  FalseBBI.IsAnalyzed = false;
1728  return false;
1729  }
1730 
1731  if (TrueBBI.BB->hasAddressTaken() || FalseBBI.BB->hasAddressTaken())
1732  // Conservatively abort if-conversion if either BB has its address taken.
1733  return false;
1734 
1735  // Put the predicated instructions from the 'true' block before the
1736  // instructions from the 'false' block, unless the true block would clobber
1737  // the predicate, in which case, do the opposite.
1738  BBInfo *BBI1 = &TrueBBI;
1739  BBInfo *BBI2 = &FalseBBI;
1740  SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
1741  if (TII->reverseBranchCondition(RevCond))
1742  llvm_unreachable("Unable to reverse branch condition!");
1743  SmallVector<MachineOperand, 4> *Cond1 = &BBI.BrCond;
1744  SmallVector<MachineOperand, 4> *Cond2 = &RevCond;
1745 
1746  // Figure out the more profitable ordering.
1747  bool DoSwap = false;
1748  if (TClobbersPred && !FClobbersPred)
1749  DoSwap = true;
1750  else if (!TClobbersPred && !FClobbersPred) {
1751  if (TrueBBI.NonPredSize > FalseBBI.NonPredSize)
1752  DoSwap = true;
1753  } else if (TClobbersPred && FClobbersPred)
1754  llvm_unreachable("Predicate info cannot be clobbered by both sides.");
1755  if (DoSwap) {
1756  std::swap(BBI1, BBI2);
1757  std::swap(Cond1, Cond2);
1758  }
1759 
1760  // Remove the conditional branch from entry to the blocks.
1761  BBI.NonPredSize -= TII->removeBranch(*BBI.BB);
1762 
1763  MachineBasicBlock &MBB1 = *BBI1->BB;
1764  MachineBasicBlock &MBB2 = *BBI2->BB;
1765 
1766  // Initialize the Redefs:
1767  // - BB2 live-in regs need implicit uses before being redefined by BB1
1768  // instructions.
1769  // - BB1 live-out regs need implicit uses before being redefined by BB2
1770  // instructions. We start with BB1 live-ins so we have the live-out regs
1771  // after tracking the BB1 instructions.
1772  Redefs.init(*TRI);
1773  if (MRI->tracksLiveness()) {
1774  Redefs.addLiveIns(MBB1);
1775  Redefs.addLiveIns(MBB2);
1776  }
1777 
1778  // Remove the duplicated instructions at the beginnings of both paths.
1779  // Skip dbg_value instructions
1782  BBI1->NonPredSize -= NumDups1;
1783  BBI2->NonPredSize -= NumDups1;
1784 
1785  // Skip past the dups on each side separately since there may be
1786  // differing dbg_value entries.
1787  for (unsigned i = 0; i < NumDups1; ++DI1) {
1788  if (!DI1->isDebugValue())
1789  ++i;
1790  }
1791  while (NumDups1 != 0) {
1792  ++DI2;
1793  if (!DI2->isDebugValue())
1794  --NumDups1;
1795  }
1796 
1797  // Compute a set of registers which must not be killed by instructions in BB1:
1798  // This is everything used+live in BB2 after the duplicated instructions. We
1799  // can compute this set by simulating liveness backwards from the end of BB2.
1800  DontKill.init(*TRI);
1801  if (MRI->tracksLiveness()) {
1802  for (const MachineInstr &MI : make_range(MBB2.rbegin(), ++DI2.getReverse()))
1803  DontKill.stepBackward(MI);
1804 
1805  for (const MachineInstr &MI : make_range(MBB1.begin(), DI1)) {
1807  Redefs.stepForward(MI, Dummy);
1808  }
1809  }
1810  BBI.BB->splice(BBI.BB->end(), &MBB1, MBB1.begin(), DI1);
1811  MBB2.erase(MBB2.begin(), DI2);
1812 
1813  // The branches have been checked to match, so it is safe to remove the branch
1814  // in BB1 and rely on the copy in BB2
1815 #ifndef NDEBUG
1816  // Unanalyzable branches must match exactly. Check that now.
1817  if (!BBI1->IsBrAnalyzable)
1818  verifySameBranchInstructions(&MBB1, &MBB2);
1819 #endif
1820  BBI1->NonPredSize -= TII->removeBranch(*BBI1->BB);
1821  // Remove duplicated instructions.
1822  DI1 = MBB1.end();
1823  for (unsigned i = 0; i != NumDups2; ) {
1824  // NumDups2 only counted non-dbg_value instructions, so this won't
1825  // run off the head of the list.
1826  assert(DI1 != MBB1.begin());
1827  --DI1;
1828  // skip dbg_value instructions
1829  if (!DI1->isDebugValue())
1830  ++i;
1831  }
1832  MBB1.erase(DI1, MBB1.end());
1833 
1834  // Kill flags in the true block for registers living into the false block
1835  // must be removed.
1836  RemoveKills(MBB1.begin(), MBB1.end(), DontKill, *TRI);
1837 
1838  DI2 = BBI2->BB->end();
1839  // The branches have been checked to match. Skip over the branch in the false
1840  // block so that we don't try to predicate it.
1841  if (RemoveBranch)
1842  BBI2->NonPredSize -= TII->removeBranch(*BBI2->BB);
1843  else {
1844  do {
1845  assert(DI2 != MBB2.begin());
1846  DI2--;
1847  } while (DI2->isBranch() || DI2->isDebugValue());
1848  DI2++;
1849  }
1850  while (NumDups2 != 0) {
1851  // NumDups2 only counted non-dbg_value instructions, so this won't
1852  // run off the head of the list.
1853  assert(DI2 != MBB2.begin());
1854  --DI2;
1855  // skip dbg_value instructions
1856  if (!DI2->isDebugValue())
1857  --NumDups2;
1858  }
1859 
1860  // Remember which registers would later be defined by the false block.
1861  // This allows us not to predicate instructions in the true block that would
1862  // later be re-defined. That is, rather than
1863  // subeq r0, r1, #1
1864  // addne r0, r1, #1
1865  // generate:
1866  // sub r0, r1, #1
1867  // addne r0, r1, #1
1868  SmallSet<unsigned, 4> RedefsByFalse;
1869  SmallSet<unsigned, 4> ExtUses;
1870  if (TII->isProfitableToUnpredicate(MBB1, MBB2)) {
1871  for (const MachineInstr &FI : make_range(MBB2.begin(), DI2)) {
1872  if (FI.isDebugValue())
1873  continue;
1875  for (const MachineOperand &MO : FI.operands()) {
1876  if (!MO.isReg())
1877  continue;
1878  unsigned Reg = MO.getReg();
1879  if (!Reg)
1880  continue;
1881  if (MO.isDef()) {
1882  Defs.push_back(Reg);
1883  } else if (!RedefsByFalse.count(Reg)) {
1884  // These are defined before ctrl flow reach the 'false' instructions.
1885  // They cannot be modified by the 'true' instructions.
1886  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
1887  SubRegs.isValid(); ++SubRegs)
1888  ExtUses.insert(*SubRegs);
1889  }
1890  }
1891 
1892  for (unsigned Reg : Defs) {
1893  if (!ExtUses.count(Reg)) {
1894  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
1895  SubRegs.isValid(); ++SubRegs)
1896  RedefsByFalse.insert(*SubRegs);
1897  }
1898  }
1899  }
1900  }
1901 
1902  // Predicate the 'true' block.
1903  PredicateBlock(*BBI1, MBB1.end(), *Cond1, &RedefsByFalse);
1904 
1905  // After predicating BBI1, if there is a predicated terminator in BBI1 and
1906  // a non-predicated in BBI2, then we don't want to predicate the one from
1907  // BBI2. The reason is that if we merged these blocks, we would end up with
1908  // two predicated terminators in the same block.
1909  if (!MBB2.empty() && (DI2 == MBB2.end())) {
1912  if (BBI1T != MBB1.end() && TII->isPredicated(*BBI1T) &&
1913  BBI2T != MBB2.end() && !TII->isPredicated(*BBI2T))
1914  --DI2;
1915  }
1916 
1917  // Predicate the 'false' block.
1918  PredicateBlock(*BBI2, DI2, *Cond2);
1919 
1920  // Merge the true block into the entry of the diamond.
1921  MergeBlocks(BBI, *BBI1, MergeAddEdges);
1922  MergeBlocks(BBI, *BBI2, MergeAddEdges);
1923  return true;
1924 }
1925 
1926 /// If convert an almost-diamond sub-CFG where the true
1927 /// and false blocks share a common tail.
1928 bool IfConverter::IfConvertForkedDiamond(
1929  BBInfo &BBI, IfcvtKind Kind,
1930  unsigned NumDups1, unsigned NumDups2,
1931  bool TClobbersPred, bool FClobbersPred) {
1932  BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1933  BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1934 
1935  // Save the debug location for later.
1936  DebugLoc dl;
1937  MachineBasicBlock::iterator TIE = TrueBBI.BB->getFirstTerminator();
1938  if (TIE != TrueBBI.BB->end())
1939  dl = TIE->getDebugLoc();
1940  // Removing branches from both blocks is safe, because we have already
1941  // determined that both blocks have the same branch instructions. The branch
1942  // will be added back at the end, unpredicated.
1943  if (!IfConvertDiamondCommon(
1944  BBI, TrueBBI, FalseBBI,
1945  NumDups1, NumDups2,
1946  TClobbersPred, FClobbersPred,
1947  /* RemoveBranch */ true, /* MergeAddEdges */ true))
1948  return false;
1949 
1950  // Add back the branch.
1951  // Debug location saved above when removing the branch from BBI2
1952  TII->insertBranch(*BBI.BB, TrueBBI.TrueBB, TrueBBI.FalseBB,
1953  TrueBBI.BrCond, dl);
1954 
1955  RemoveExtraEdges(BBI);
1956 
1957  // Update block info.
1958  BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true;
1959  InvalidatePreds(*BBI.BB);
1960 
1961  // FIXME: Must maintain LiveIns.
1962  return true;
1963 }
1964 
1965 /// If convert a diamond sub-CFG.
1966 bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
1967  unsigned NumDups1, unsigned NumDups2,
1968  bool TClobbersPred, bool FClobbersPred) {
1969  BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1970  BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1971  MachineBasicBlock *TailBB = TrueBBI.TrueBB;
1972 
1973  // True block must fall through or end with an unanalyzable terminator.
1974  if (!TailBB) {
1975  if (blockAlwaysFallThrough(TrueBBI))
1976  TailBB = FalseBBI.TrueBB;
1977  assert((TailBB || !TrueBBI.IsBrAnalyzable) && "Unexpected!");
1978  }
1979 
1980  if (!IfConvertDiamondCommon(
1981  BBI, TrueBBI, FalseBBI,
1982  NumDups1, NumDups2,
1983  TClobbersPred, FClobbersPred,
1984  /* RemoveBranch */ TrueBBI.IsBrAnalyzable,
1985  /* MergeAddEdges */ TailBB == nullptr))
1986  return false;
1987 
1988  // If the if-converted block falls through or unconditionally branches into
1989  // the tail block, and the tail block does not have other predecessors, then
1990  // fold the tail block in as well. Otherwise, unless it falls through to the
1991  // tail, add a unconditional branch to it.
1992  if (TailBB) {
1993  BBInfo &TailBBI = BBAnalysis[TailBB->getNumber()];
1994  bool CanMergeTail = !TailBBI.HasFallThrough &&
1995  !TailBBI.BB->hasAddressTaken();
1996  // The if-converted block can still have a predicated terminator
1997  // (e.g. a predicated return). If that is the case, we cannot merge
1998  // it with the tail block.
1999  MachineBasicBlock::const_iterator TI = BBI.BB->getFirstTerminator();
2000  if (TI != BBI.BB->end() && TII->isPredicated(*TI))
2001  CanMergeTail = false;
2002  // There may still be a fall-through edge from BBI1 or BBI2 to TailBB;
2003  // check if there are any other predecessors besides those.
2004  unsigned NumPreds = TailBB->pred_size();
2005  if (NumPreds > 1)
2006  CanMergeTail = false;
2007  else if (NumPreds == 1 && CanMergeTail) {
2009  if (*PI != TrueBBI.BB && *PI != FalseBBI.BB)
2010  CanMergeTail = false;
2011  }
2012  if (CanMergeTail) {
2013  MergeBlocks(BBI, TailBBI);
2014  TailBBI.IsDone = true;
2015  } else {
2016  BBI.BB->addSuccessor(TailBB, BranchProbability::getOne());
2017  InsertUncondBranch(*BBI.BB, *TailBB, TII);
2018  BBI.HasFallThrough = false;
2019  }
2020  }
2021 
2022  // RemoveExtraEdges won't work if the block has an unanalyzable branch,
2023  // which can happen here if TailBB is unanalyzable and is merged, so
2024  // explicitly remove BBI1 and BBI2 as successors.
2025  BBI.BB->removeSuccessor(TrueBBI.BB);
2026  BBI.BB->removeSuccessor(FalseBBI.BB, /* NormalizeSuccessProbs */ true);
2027  RemoveExtraEdges(BBI);
2028 
2029  // Update block info.
2030  BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true;
2031  InvalidatePreds(*BBI.BB);
2032 
2033  // FIXME: Must maintain LiveIns.
2034  return true;
2035 }
2036 
2037 static bool MaySpeculate(const MachineInstr &MI,
2038  SmallSet<unsigned, 4> &LaterRedefs) {
2039  bool SawStore = true;
2040  if (!MI.isSafeToMove(nullptr, SawStore))
2041  return false;
2042 
2043  for (const MachineOperand &MO : MI.operands()) {
2044  if (!MO.isReg())
2045  continue;
2046  unsigned Reg = MO.getReg();
2047  if (!Reg)
2048  continue;
2049  if (MO.isDef() && !LaterRedefs.count(Reg))
2050  return false;
2051  }
2052 
2053  return true;
2054 }
2055 
2056 /// Predicate instructions from the start of the block to the specified end with
2057 /// the specified condition.
2058 void IfConverter::PredicateBlock(BBInfo &BBI,
2061  SmallSet<unsigned, 4> *LaterRedefs) {
2062  bool AnyUnpred = false;
2063  bool MaySpec = LaterRedefs != nullptr;
2064  for (MachineInstr &I : make_range(BBI.BB->begin(), E)) {
2065  if (I.isDebugValue() || TII->isPredicated(I))
2066  continue;
2067  // It may be possible not to predicate an instruction if it's the 'true'
2068  // side of a diamond and the 'false' side may re-define the instruction's
2069  // defs.
2070  if (MaySpec && MaySpeculate(I, *LaterRedefs)) {
2071  AnyUnpred = true;
2072  continue;
2073  }
2074  // If any instruction is predicated, then every instruction after it must
2075  // be predicated.
2076  MaySpec = false;
2077  if (!TII->PredicateInstruction(I, Cond)) {
2078 #ifndef NDEBUG
2079  dbgs() << "Unable to predicate " << I << "!\n";
2080 #endif
2081  llvm_unreachable(nullptr);
2082  }
2083 
2084  // If the predicated instruction now redefines a register as the result of
2085  // if-conversion, add an implicit kill.
2086  UpdatePredRedefs(I, Redefs);
2087  }
2088 
2089  BBI.Predicate.append(Cond.begin(), Cond.end());
2090 
2091  BBI.IsAnalyzed = false;
2092  BBI.NonPredSize = 0;
2093 
2094  ++NumIfConvBBs;
2095  if (AnyUnpred)
2096  ++NumUnpred;
2097 }
2098 
2099 /// Copy and predicate instructions from source BB to the destination block.
2100 /// Skip end of block branches if IgnoreBr is true.
2101 void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
2103  bool IgnoreBr) {
2104  MachineFunction &MF = *ToBBI.BB->getParent();
2105 
2106  MachineBasicBlock &FromMBB = *FromBBI.BB;
2107  for (MachineInstr &I : FromMBB) {
2108  // Do not copy the end of the block branches.
2109  if (IgnoreBr && I.isBranch())
2110  break;
2111 
2112  MachineInstr *MI = MF.CloneMachineInstr(&I);
2113  ToBBI.BB->insert(ToBBI.BB->end(), MI);
2114  ToBBI.NonPredSize++;
2115  unsigned ExtraPredCost = TII->getPredicationCost(I);
2116  unsigned NumCycles = SchedModel.computeInstrLatency(&I, false);
2117  if (NumCycles > 1)
2118  ToBBI.ExtraCost += NumCycles-1;
2119  ToBBI.ExtraCost2 += ExtraPredCost;
2120 
2121  if (!TII->isPredicated(I) && !MI->isDebugValue()) {
2122  if (!TII->PredicateInstruction(*MI, Cond)) {
2123 #ifndef NDEBUG
2124  dbgs() << "Unable to predicate " << I << "!\n";
2125 #endif
2126  llvm_unreachable(nullptr);
2127  }
2128  }
2129 
2130  // If the predicated instruction now redefines a register as the result of
2131  // if-conversion, add an implicit kill.
2132  UpdatePredRedefs(*MI, Redefs);
2133 
2134  // Some kill flags may not be correct anymore.
2135  if (!DontKill.empty())
2136  RemoveKills(*MI, DontKill);
2137  }
2138 
2139  if (!IgnoreBr) {
2140  std::vector<MachineBasicBlock *> Succs(FromMBB.succ_begin(),
2141  FromMBB.succ_end());
2142  MachineBasicBlock *NBB = getNextBlock(FromMBB);
2143  MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr;
2144 
2145  for (MachineBasicBlock *Succ : Succs) {
2146  // Fallthrough edge can't be transferred.
2147  if (Succ == FallThrough)
2148  continue;
2149  ToBBI.BB->addSuccessor(Succ);
2150  }
2151  }
2152 
2153  ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
2154  ToBBI.Predicate.append(Cond.begin(), Cond.end());
2155 
2156  ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
2157  ToBBI.IsAnalyzed = false;
2158 
2159  ++NumDupBBs;
2160 }
2161 
2162 /// Move all instructions from FromBB to the end of ToBB. This will leave
2163 /// FromBB as an empty block, so remove all of its successor edges except for
2164 /// the fall-through edge. If AddEdges is true, i.e., when FromBBI's branch is
2165 /// being moved, add those successor edges to ToBBI.
2166 void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) {
2167  MachineBasicBlock &FromMBB = *FromBBI.BB;
2168  assert(!FromMBB.hasAddressTaken() &&
2169  "Removing a BB whose address is taken!");
2170 
2171  // In case FromMBB contains terminators (e.g. return instruction),
2172  // first move the non-terminator instructions, then the terminators.
2174  MachineBasicBlock::iterator ToTI = ToBBI.BB->getFirstTerminator();
2175  ToBBI.BB->splice(ToTI, &FromMBB, FromMBB.begin(), FromTI);
2176 
2177  // If FromBB has non-predicated terminator we should copy it at the end.
2178  if (FromTI != FromMBB.end() && !TII->isPredicated(*FromTI))
2179  ToTI = ToBBI.BB->end();
2180  ToBBI.BB->splice(ToTI, &FromMBB, FromTI, FromMBB.end());
2181 
2182  // Force normalizing the successors' probabilities of ToBBI.BB to convert all
2183  // unknown probabilities into known ones.
2184  // FIXME: This usage is too tricky and in the future we would like to
2185  // eliminate all unknown probabilities in MBB.
2186  ToBBI.BB->normalizeSuccProbs();
2187 
2188  SmallVector<MachineBasicBlock *, 4> FromSuccs(FromMBB.succ_begin(),
2189  FromMBB.succ_end());
2190  MachineBasicBlock *NBB = getNextBlock(FromMBB);
2191  MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr;
2192  // The edge probability from ToBBI.BB to FromMBB, which is only needed when
2193  // AddEdges is true and FromMBB is a successor of ToBBI.BB.
2194  auto To2FromProb = BranchProbability::getZero();
2195  if (AddEdges && ToBBI.BB->isSuccessor(&FromMBB)) {
2196  To2FromProb = MBPI->getEdgeProbability(ToBBI.BB, &FromMBB);
2197  // Set the edge probability from ToBBI.BB to FromMBB to zero to avoid the
2198  // edge probability being merged to other edges when this edge is removed
2199  // later.
2200  ToBBI.BB->setSuccProbability(find(ToBBI.BB->successors(), &FromMBB),
2202  }
2203 
2204  for (MachineBasicBlock *Succ : FromSuccs) {
2205  // Fallthrough edge can't be transferred.
2206  if (Succ == FallThrough)
2207  continue;
2208 
2209  auto NewProb = BranchProbability::getZero();
2210  if (AddEdges) {
2211  // Calculate the edge probability for the edge from ToBBI.BB to Succ,
2212  // which is a portion of the edge probability from FromMBB to Succ. The
2213  // portion ratio is the edge probability from ToBBI.BB to FromMBB (if
2214  // FromBBI is a successor of ToBBI.BB. See comment below for excepion).
2215  NewProb = MBPI->getEdgeProbability(&FromMBB, Succ);
2216 
2217  // To2FromProb is 0 when FromMBB is not a successor of ToBBI.BB. This
2218  // only happens when if-converting a diamond CFG and FromMBB is the
2219  // tail BB. In this case FromMBB post-dominates ToBBI.BB and hence we
2220  // could just use the probabilities on FromMBB's out-edges when adding
2221  // new successors.
2222  if (!To2FromProb.isZero())
2223  NewProb *= To2FromProb;
2224  }
2225 
2226  FromMBB.removeSuccessor(Succ);
2227 
2228  if (AddEdges) {
2229  // If the edge from ToBBI.BB to Succ already exists, update the
2230  // probability of this edge by adding NewProb to it. An example is shown
2231  // below, in which A is ToBBI.BB and B is FromMBB. In this case we
2232  // don't have to set C as A's successor as it already is. We only need to
2233  // update the edge probability on A->C. Note that B will not be
2234  // immediately removed from A's successors. It is possible that B->D is
2235  // not removed either if D is a fallthrough of B. Later the edge A->D
2236  // (generated here) and B->D will be combined into one edge. To maintain
2237  // correct edge probability of this combined edge, we need to set the edge
2238  // probability of A->B to zero, which is already done above. The edge
2239  // probability on A->D is calculated by scaling the original probability
2240  // on A->B by the probability of B->D.
2241  //
2242  // Before ifcvt: After ifcvt (assume B->D is kept):
2243  //
2244  // A A
2245  // /| /|\
2246  // / B / B|
2247  // | /| | ||
2248  // |/ | | |/
2249  // C D C D
2250  //
2251  if (ToBBI.BB->isSuccessor(Succ))
2252  ToBBI.BB->setSuccProbability(
2253  find(ToBBI.BB->successors(), Succ),
2254  MBPI->getEdgeProbability(ToBBI.BB, Succ) + NewProb);
2255  else
2256  ToBBI.BB->addSuccessor(Succ, NewProb);
2257  }
2258  }
2259 
2260  // Now FromBBI always falls through to the next block!
2261  if (NBB && !FromMBB.isSuccessor(NBB))
2262  FromMBB.addSuccessor(NBB);
2263 
2264  // Normalize the probabilities of ToBBI.BB's successors with all adjustment
2265  // we've done above.
2266  ToBBI.BB->normalizeSuccProbs();
2267 
2268  ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
2269  FromBBI.Predicate.clear();
2270 
2271  ToBBI.NonPredSize += FromBBI.NonPredSize;
2272  ToBBI.ExtraCost += FromBBI.ExtraCost;
2273  ToBBI.ExtraCost2 += FromBBI.ExtraCost2;
2274  FromBBI.NonPredSize = 0;
2275  FromBBI.ExtraCost = 0;
2276  FromBBI.ExtraCost2 = 0;
2277 
2278  ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
2279  ToBBI.HasFallThrough = FromBBI.HasFallThrough;
2280  ToBBI.IsAnalyzed = false;
2281  FromBBI.IsAnalyzed = false;
2282 }
2283 
2284 FunctionPass *
2286  return new IfConverter(std::move(Ftor));
2287 }
void push_back(const T &Elt)
Definition: SmallVector.h:211
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
bool DefinesPredicate(MachineInstr &MI, std::vector< MachineOperand > &Pred) const override
If the specified instruction defines any predicate or condition code register(s) used for predication...
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
uint64_t Token
STATISTIC(NumFunctions,"Total number of functions")
size_t i
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
static void UpdatePredRedefs(MachineInstr &MI, LivePhysRegs &Redefs)
Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all values defined in MI whic...
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
std::pair< iterator, bool > insert(const ValueT &Val)
insert - Attempts to insert a new element.
Definition: SparseSet.h:249
static cl::opt< bool > DisableDiamond("disable-ifcvt-diamond", cl::init(false), cl::Hidden)
static cl::opt< bool > DisableForkedDiamond("disable-ifcvt-forked-diamond", cl::init(false), cl::Hidden)
static cl::opt< bool > DisableTriangleFR("disable-ifcvt-triangle-false-rev", cl::init(false), cl::Hidden)
size_type count(const KeyT &Key) const
count - Returns 1 if this set contains an element identified by Key, 0 otherwise. ...
Definition: SparseSet.h:235
iterator getFirstNonDebugInstr()
Returns an iterator to the first non-debug instruction in the basic block, or end().
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool SubsumesPredicate(ArrayRef< MachineOperand > Pred1, ArrayRef< MachineOperand > Pred2) const override
Returns true if the first specified predicate subsumes the second, e.g.
MIBundleOperands - Iterate over all operands in a bundle of machine instructions. ...
bool isDead() const
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
Reverses the branch condition of the specified condition list, returning false on success and true if...
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:47
FunctionPass * createIfConverter(std::function< bool(const MachineFunction &)> Ftor)
A debug info location.
Definition: DebugLoc.h:34
void setIsDead(bool Val=true)
static cl::opt< bool > DisableSimple("disable-ifcvt-simple", cl::init(false), cl::Hidden)
static BranchProbability getOne()
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:301
iterator_range< succ_iterator > successors()
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
bool OptimizeFunction(MachineFunction &MF, const TargetInstrInfo *tii, const TargetRegisterInfo *tri, MachineModuleInfo *mmi, MachineLoopInfo *mli=nullptr, bool AfterPlacement=false)
OptimizeFunction - Perhaps branch folding, tail merging and other CFG optimizations on the given func...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Provide an instruction scheduling machine model to CodeGen passes.
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
void initializeIfConverterPass(PassRegistry &)
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
bool PredicateInstruction(MachineInstr &MI, ArrayRef< MachineOperand > Cond) const override
Convert the instruction into a predicated instruction.
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
const HexagonRegisterInfo & getRegisterInfo() const
HexagonInstrInfo specifics.
MachineBasicBlock * MBB
Function Alias Analysis false
std::vector< MachineBasicBlock * >::iterator pred_iterator
static MachineBasicBlock * getNextBlock(MachineBasicBlock &MBB)
Returns the next block in the function blocks ordering.
reverse_iterator rbegin()
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:131
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:777
bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, BranchProbability Probability) const override
Return true if it's profitable for if-converter to duplicate instructions of specified accumulated in...
static cl::opt< bool > DisableTriangleR("disable-ifcvt-triangle-rev", cl::init(false), cl::Hidden)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
static MachineBasicBlock * findFalseBlock(MachineBasicBlock *BB, MachineBasicBlock *TrueBB)
BB has a fallthrough. Find its 'false' successor given its 'true' successor.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
static cl::opt< bool > DisableTriangleF("disable-ifcvt-triangle-false", cl::init(false), cl::Hidden)
Represent the analysis usage information of a pass.
static bool canFallThroughTo(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB)
Returns true either if ToMBB is the next block after MBB or that all the intervening blocks are empty...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
bool contains(unsigned Reg) const
Returns true if register Reg is contained in the set.
Definition: LivePhysRegs.h:102
static const unsigned End
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
self_iterator getIterator()
Definition: ilist_node.h:81
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
void setUniverse(unsigned U)
setUniverse - Set the universe size which determines the largest key the set can hold.
Definition: SparseSet.h:155
iterator_range< pred_iterator > predecessors()
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
MCSubRegIterator enumerates all sub-registers of Reg.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isPredicated(const MachineInstr &MI) const override
Returns true if the instruction is already predicated.
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:64
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Iterator for intrusive lists based on ilist_node.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:757
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
CloneMachineInstr - Create a new MachineInstr which is a copy of the 'Orig' instruction, identical in all ways except the instruction has no parent, prev, or next.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
static cl::opt< bool > DisableTriangle("disable-ifcvt-triangle", cl::init(false), cl::Hidden)
virtual const TargetLowering * getTargetLowering() const
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:27
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:50
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
bool isPredicable(MachineInstr &MI) const override
Return true if the specified instruction can be predicated.
static bool MaySpeculate(const MachineInstr &MI, SmallSet< unsigned, 4 > &LaterRedefs)
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, unsigned ExtraPredCycles, BranchProbability Probability) const override
Return true if it's profitable to predicate instructions with accumulated instruction latency of "Num...
static void RemoveKills(MachineInstr &MI, const LivePhysRegs &DontKill)
Remove kill flags from operands with a registers in the DontKill set.
static cl::opt< int > IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden)
IterT skipDebugInstructionsBackward(IterT It, IterT Begin)
Decrement It until it points to a non-debug instruction or to Begin and return the resulting iterator...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
Remove the branching code at the end of the specific MBB.
Maximum length of the test input If
bool isPredicated(MCInstrInfo const &MCII, MCInst const &MCI)
IterT skipDebugInstructionsForward(IterT It, IterT End)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator...
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
TargetSubtargetInfo - Generic base class for all target subtargets.
Representation of each machine instruction.
Definition: MachineInstr.h:52
bool hasAddressTaken() const
Test whether this block is potentially the target of an indirect branch.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
static cl::opt< int > IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden)
A set of live physical registers with functions to track liveness when walking backward/forward throu...
Definition: LivePhysRegs.h:45
static cl::opt< bool > IfCvtBranchFold("ifcvt-branch-fold", cl::init(true), cl::Hidden)
char & IfConverterID
IfConverter - This pass performs machine code if conversion.
#define I(x, y, z)
Definition: MD5.cpp:54
static cl::opt< bool > DisableSimpleF("disable-ifcvt-simple-false", cl::init(false), cl::Hidden)
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const unsigned Kind
unsigned getReg() const
getReg - Returns the register number.
bool isValid() const
isValid - Returns true until all the operands have been visited.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual const TargetInstrInfo * getInstrInfo() const
static void InsertUncondBranch(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB, const TargetInstrInfo *TII)
Inserts an unconditional branch from MBB to ToMBB.
#define DEBUG(X)
Definition: Debug.h:100
This class keeps track of branch frequencies of newly created blocks and tail-merged blocks...
print Print MemDeps of function
IRTranslator LLVM IR MI
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
static BranchProbability getZero()
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
unsigned pred_size() const
Properties which a MachineFunction may have at a given point in time.
This file describes how to lower LLVM code to machine code.
static void verifySameBranchInstructions(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2)
early Early If Converter
static cl::opt< int > IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden)
BranchProbability getCompl() const
static void shrinkInclusiveRange(MachineBasicBlock::iterator &Begin, MachineBasicBlock::iterator &It, bool &Empty)
Shrink the provided inclusive range by one instruction.