LLVM  4.0.0
MachineSink.cpp
Go to the documentation of this file.
1 //===-- MachineSink.cpp - Sinking for machine instructions ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass moves instructions into successor blocks when possible, so that
11 // they aren't executed on paths where their results aren't needed.
12 //
13 // This pass is not intended to be a replacement or a complete alternative
14 // for an LLVM-IR-level sinking pass. It is only designed to sink simple
15 // constructs that are not exposed before lowering and instruction selection.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "llvm/CodeGen/Passes.h"
20 #include "llvm/ADT/SetVector.h"
21 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/Statistic.h"
36 #include "llvm/IR/LLVMContext.h"
38 #include "llvm/Support/Debug.h"
43 #include <algorithm>
44 #include <cassert>
45 #include <cstdint>
46 #include <map>
47 #include <utility>
48 #include <vector>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "machine-sink"
53 
54 static cl::opt<bool>
55 SplitEdges("machine-sink-split",
56  cl::desc("Split critical edges during machine sinking"),
57  cl::init(true), cl::Hidden);
58 
59 static cl::opt<bool>
60 UseBlockFreqInfo("machine-sink-bfi",
61  cl::desc("Use block frequency info to find successors to sink"),
62  cl::init(true), cl::Hidden);
63 
65  "machine-sink-split-probability-threshold",
66  cl::desc(
67  "Percentage threshold for splitting single-instruction critical edge. "
68  "If the branch threshold is higher than this threshold, we allow "
69  "speculative execution of up to 1 instruction to avoid branching to "
70  "splitted critical edge"),
71  cl::init(40), cl::Hidden);
72 
73 STATISTIC(NumSunk, "Number of machine instructions sunk");
74 STATISTIC(NumSplit, "Number of critical edges split");
75 STATISTIC(NumCoalesces, "Number of copies coalesced");
76 
77 namespace {
78 
79  class MachineSinking : public MachineFunctionPass {
80  const TargetInstrInfo *TII;
81  const TargetRegisterInfo *TRI;
82  MachineRegisterInfo *MRI; // Machine register information
83  MachineDominatorTree *DT; // Machine dominator tree
84  MachinePostDominatorTree *PDT; // Machine post dominator tree
85  MachineLoopInfo *LI;
86  const MachineBlockFrequencyInfo *MBFI;
87  const MachineBranchProbabilityInfo *MBPI;
88  AliasAnalysis *AA;
89 
90  // Remember which edges have been considered for breaking.
92  CEBCandidates;
93  // Remember which edges we are about to split.
94  // This is different from CEBCandidates since those edges
95  // will be split.
97 
98  SparseBitVector<> RegsToClearKillFlags;
99 
100  typedef std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>
101  AllSuccsCache;
102 
103  public:
104  static char ID; // Pass identification
105 
106  MachineSinking() : MachineFunctionPass(ID) {
108  }
109 
110  bool runOnMachineFunction(MachineFunction &MF) override;
111 
112  void getAnalysisUsage(AnalysisUsage &AU) const override {
113  AU.setPreservesCFG();
123  if (UseBlockFreqInfo)
125  }
126 
127  void releaseMemory() override {
128  CEBCandidates.clear();
129  }
130 
131  private:
133  bool isWorthBreakingCriticalEdge(MachineInstr &MI,
134  MachineBasicBlock *From,
135  MachineBasicBlock *To);
136  /// \brief Postpone the splitting of the given critical
137  /// edge (\p From, \p To).
138  ///
139  /// We do not split the edges on the fly. Indeed, this invalidates
140  /// the dominance information and thus triggers a lot of updates
141  /// of that information underneath.
142  /// Instead, we postpone all the splits after each iteration of
143  /// the main loop. That way, the information is at least valid
144  /// for the lifetime of an iteration.
145  ///
146  /// \return True if the edge is marked as toSplit, false otherwise.
147  /// False can be returned if, for instance, this is not profitable.
148  bool PostponeSplitCriticalEdge(MachineInstr &MI,
149  MachineBasicBlock *From,
150  MachineBasicBlock *To,
151  bool BreakPHIEdge);
153  AllSuccsCache &AllSuccessors);
155  MachineBasicBlock *DefMBB,
156  bool &BreakPHIEdge, bool &LocalUse) const;
158  bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
159  bool isProfitableToSinkTo(unsigned Reg, MachineInstr &MI,
161  MachineBasicBlock *SuccToSinkTo,
162  AllSuccsCache &AllSuccessors);
163 
164  bool PerformTrivialForwardCoalescing(MachineInstr &MI,
166 
168  GetAllSortedSuccessors(MachineInstr &MI, MachineBasicBlock *MBB,
169  AllSuccsCache &AllSuccessors) const;
170  };
171 
172 } // end anonymous namespace
173 
174 char MachineSinking::ID = 0;
176 INITIALIZE_PASS_BEGIN(MachineSinking, "machine-sink",
177  "Machine code sinking", false, false)
182 INITIALIZE_PASS_END(MachineSinking, "machine-sink",
183  "Machine code sinking", false, false)
184 
185 bool MachineSinking::PerformTrivialForwardCoalescing(MachineInstr &MI,
187  if (!MI.isCopy())
188  return false;
189 
190  unsigned SrcReg = MI.getOperand(1).getReg();
191  unsigned DstReg = MI.getOperand(0).getReg();
194  !MRI->hasOneNonDBGUse(SrcReg))
195  return false;
196 
197  const TargetRegisterClass *SRC = MRI->getRegClass(SrcReg);
198  const TargetRegisterClass *DRC = MRI->getRegClass(DstReg);
199  if (SRC != DRC)
200  return false;
201 
202  MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
203  if (DefMI->isCopyLike())
204  return false;
205  DEBUG(dbgs() << "Coalescing: " << *DefMI);
206  DEBUG(dbgs() << "*** to: " << MI);
207  MRI->replaceRegWith(DstReg, SrcReg);
208  MI.eraseFromParent();
209 
210  // Conservatively, clear any kill flags, since it's possible that they are no
211  // longer correct.
212  MRI->clearKillFlags(SrcReg);
213 
214  ++NumCoalesces;
215  return true;
216 }
217 
218 /// AllUsesDominatedByBlock - Return true if all uses of the specified register
219 /// occur in blocks dominated by the specified block. If any use is in the
220 /// definition block, then return false since it is never legal to move def
221 /// after uses.
222 bool
225  MachineBasicBlock *DefMBB,
226  bool &BreakPHIEdge,
227  bool &LocalUse) const {
229  "Only makes sense for vregs");
230 
231  // Ignore debug uses because debug info doesn't affect the code.
232  if (MRI->use_nodbg_empty(Reg))
233  return true;
234 
235  // BreakPHIEdge is true if all the uses are in the successor MBB being sunken
236  // into and they are all PHI nodes. In this case, machine-sink must break
237  // the critical edge first. e.g.
238  //
239  // BB#1: derived from LLVM BB %bb4.preheader
240  // Predecessors according to CFG: BB#0
241  // ...
242  // %reg16385<def> = DEC64_32r %reg16437, %EFLAGS<imp-def,dead>
243  // ...
244  // JE_4 <BB#37>, %EFLAGS<imp-use>
245  // Successors according to CFG: BB#37 BB#2
246  //
247  // BB#2: derived from LLVM BB %bb.nph
248  // Predecessors according to CFG: BB#0 BB#1
249  // %reg16386<def> = PHI %reg16434, <BB#0>, %reg16385, <BB#1>
250  BreakPHIEdge = true;
251  for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
252  MachineInstr *UseInst = MO.getParent();
253  unsigned OpNo = &MO - &UseInst->getOperand(0);
254  MachineBasicBlock *UseBlock = UseInst->getParent();
255  if (!(UseBlock == MBB && UseInst->isPHI() &&
256  UseInst->getOperand(OpNo+1).getMBB() == DefMBB)) {
257  BreakPHIEdge = false;
258  break;
259  }
260  }
261  if (BreakPHIEdge)
262  return true;
263 
264  for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
265  // Determine the block of the use.
266  MachineInstr *UseInst = MO.getParent();
267  unsigned OpNo = &MO - &UseInst->getOperand(0);
268  MachineBasicBlock *UseBlock = UseInst->getParent();
269  if (UseInst->isPHI()) {
270  // PHI nodes use the operand in the predecessor block, not the block with
271  // the PHI.
272  UseBlock = UseInst->getOperand(OpNo+1).getMBB();
273  } else if (UseBlock == DefMBB) {
274  LocalUse = true;
275  return false;
276  }
277 
278  // Check that it dominates.
279  if (!DT->dominates(MBB, UseBlock))
280  return false;
281  }
282 
283  return true;
284 }
285 
286 bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
287  if (skipFunction(*MF.getFunction()))
288  return false;
289 
290  DEBUG(dbgs() << "******** Machine Sinking ********\n");
291 
292  TII = MF.getSubtarget().getInstrInfo();
293  TRI = MF.getSubtarget().getRegisterInfo();
294  MRI = &MF.getRegInfo();
295  DT = &getAnalysis<MachineDominatorTree>();
296  PDT = &getAnalysis<MachinePostDominatorTree>();
297  LI = &getAnalysis<MachineLoopInfo>();
298  MBFI = UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() : nullptr;
299  MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
300  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
301 
302  bool EverMadeChange = false;
303 
304  while (true) {
305  bool MadeChange = false;
306 
307  // Process all basic blocks.
308  CEBCandidates.clear();
309  ToSplit.clear();
310  for (auto &MBB: MF)
311  MadeChange |= ProcessBlock(MBB);
312 
313  // If we have anything we marked as toSplit, split it now.
314  for (auto &Pair : ToSplit) {
315  auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *this);
316  if (NewSucc != nullptr) {
317  DEBUG(dbgs() << " *** Splitting critical edge:"
318  " BB#" << Pair.first->getNumber()
319  << " -- BB#" << NewSucc->getNumber()
320  << " -- BB#" << Pair.second->getNumber() << '\n');
321  MadeChange = true;
322  ++NumSplit;
323  } else
324  DEBUG(dbgs() << " *** Not legal to break critical edge\n");
325  }
326  // If this iteration over the code changed anything, keep iterating.
327  if (!MadeChange) break;
328  EverMadeChange = true;
329  }
330 
331  // Now clear any kill flags for recorded registers.
332  for (auto I : RegsToClearKillFlags)
333  MRI->clearKillFlags(I);
334  RegsToClearKillFlags.clear();
335 
336  return EverMadeChange;
337 }
338 
340  // Can't sink anything out of a block that has less than two successors.
341  if (MBB.succ_size() <= 1 || MBB.empty()) return false;
342 
343  // Don't bother sinking code out of unreachable blocks. In addition to being
344  // unprofitable, it can also lead to infinite looping, because in an
345  // unreachable loop there may be nowhere to stop.
346  if (!DT->isReachableFromEntry(&MBB)) return false;
347 
348  bool MadeChange = false;
349 
350  // Cache all successors, sorted by frequency info and loop depth.
351  AllSuccsCache AllSuccessors;
352 
353  // Walk the basic block bottom-up. Remember if we saw a store.
355  --I;
356  bool ProcessedBegin, SawStore = false;
357  do {
358  MachineInstr &MI = *I; // The instruction to sink.
359 
360  // Predecrement I (if it's not begin) so that it isn't invalidated by
361  // sinking.
362  ProcessedBegin = I == MBB.begin();
363  if (!ProcessedBegin)
364  --I;
365 
366  if (MI.isDebugValue())
367  continue;
368 
369  bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
370  if (Joined) {
371  MadeChange = true;
372  continue;
373  }
374 
375  if (SinkInstruction(MI, SawStore, AllSuccessors)) {
376  ++NumSunk;
377  MadeChange = true;
378  }
379 
380  // If we just processed the first instruction in the block, we're done.
381  } while (!ProcessedBegin);
382 
383  return MadeChange;
384 }
385 
386 bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr &MI,
387  MachineBasicBlock *From,
388  MachineBasicBlock *To) {
389  // FIXME: Need much better heuristics.
390 
391  // If the pass has already considered breaking this edge (during this pass
392  // through the function), then let's go ahead and break it. This means
393  // sinking multiple "cheap" instructions into the same block.
394  if (!CEBCandidates.insert(std::make_pair(From, To)).second)
395  return true;
396 
397  if (!MI.isCopy() && !TII->isAsCheapAsAMove(MI))
398  return true;
399 
400  if (From->isSuccessor(To) && MBPI->getEdgeProbability(From, To) <=
402  return true;
403 
404  // MI is cheap, we probably don't want to break the critical edge for it.
405  // However, if this would allow some definitions of its source operands
406  // to be sunk then it's probably worth it.
407  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
408  const MachineOperand &MO = MI.getOperand(i);
409  if (!MO.isReg() || !MO.isUse())
410  continue;
411  unsigned Reg = MO.getReg();
412  if (Reg == 0)
413  continue;
414 
415  // We don't move live definitions of physical registers,
416  // so sinking their uses won't enable any opportunities.
418  continue;
419 
420  // If this instruction is the only user of a virtual register,
421  // check if breaking the edge will enable sinking
422  // both this instruction and the defining instruction.
423  if (MRI->hasOneNonDBGUse(Reg)) {
424  // If the definition resides in same MBB,
425  // claim it's likely we can sink these together.
426  // If definition resides elsewhere, we aren't
427  // blocking it from being sunk so don't break the edge.
428  MachineInstr *DefMI = MRI->getVRegDef(Reg);
429  if (DefMI->getParent() == MI.getParent())
430  return true;
431  }
432  }
433 
434  return false;
435 }
436 
437 bool MachineSinking::PostponeSplitCriticalEdge(MachineInstr &MI,
438  MachineBasicBlock *FromBB,
439  MachineBasicBlock *ToBB,
440  bool BreakPHIEdge) {
441  if (!isWorthBreakingCriticalEdge(MI, FromBB, ToBB))
442  return false;
443 
444  // Avoid breaking back edge. From == To means backedge for single BB loop.
445  if (!SplitEdges || FromBB == ToBB)
446  return false;
447 
448  // Check for backedges of more "complex" loops.
449  if (LI->getLoopFor(FromBB) == LI->getLoopFor(ToBB) &&
450  LI->isLoopHeader(ToBB))
451  return false;
452 
453  // It's not always legal to break critical edges and sink the computation
454  // to the edge.
455  //
456  // BB#1:
457  // v1024
458  // Beq BB#3
459  // <fallthrough>
460  // BB#2:
461  // ... no uses of v1024
462  // <fallthrough>
463  // BB#3:
464  // ...
465  // = v1024
466  //
467  // If BB#1 -> BB#3 edge is broken and computation of v1024 is inserted:
468  //
469  // BB#1:
470  // ...
471  // Bne BB#2
472  // BB#4:
473  // v1024 =
474  // B BB#3
475  // BB#2:
476  // ... no uses of v1024
477  // <fallthrough>
478  // BB#3:
479  // ...
480  // = v1024
481  //
482  // This is incorrect since v1024 is not computed along the BB#1->BB#2->BB#3
483  // flow. We need to ensure the new basic block where the computation is
484  // sunk to dominates all the uses.
485  // It's only legal to break critical edge and sink the computation to the
486  // new block if all the predecessors of "To", except for "From", are
487  // not dominated by "From". Given SSA property, this means these
488  // predecessors are dominated by "To".
489  //
490  // There is no need to do this check if all the uses are PHI nodes. PHI
491  // sources are only defined on the specific predecessor edges.
492  if (!BreakPHIEdge) {
494  E = ToBB->pred_end(); PI != E; ++PI) {
495  if (*PI == FromBB)
496  continue;
497  if (!DT->dominates(ToBB, *PI))
498  return false;
499  }
500  }
501 
502  ToSplit.insert(std::make_pair(FromBB, ToBB));
503 
504  return true;
505 }
506 
507 /// collectDebgValues - Scan instructions following MI and collect any
508 /// matching DBG_VALUEs.
510  SmallVectorImpl<MachineInstr *> &DbgValues) {
511  DbgValues.clear();
512  if (!MI.getOperand(0).isReg())
513  return;
514 
515  MachineBasicBlock::iterator DI = MI; ++DI;
516  for (MachineBasicBlock::iterator DE = MI.getParent()->end();
517  DI != DE; ++DI) {
518  if (!DI->isDebugValue())
519  return;
520  if (DI->getOperand(0).isReg() &&
521  DI->getOperand(0).getReg() == MI.getOperand(0).getReg())
522  DbgValues.push_back(&*DI);
523  }
524 }
525 
526 /// isProfitableToSinkTo - Return true if it is profitable to sink MI.
527 bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr &MI,
528  MachineBasicBlock *MBB,
529  MachineBasicBlock *SuccToSinkTo,
530  AllSuccsCache &AllSuccessors) {
531  assert (SuccToSinkTo && "Invalid SinkTo Candidate BB");
532 
533  if (MBB == SuccToSinkTo)
534  return false;
535 
536  // It is profitable if SuccToSinkTo does not post dominate current block.
537  if (!PDT->dominates(SuccToSinkTo, MBB))
538  return true;
539 
540  // It is profitable to sink an instruction from a deeper loop to a shallower
541  // loop, even if the latter post-dominates the former (PR21115).
542  if (LI->getLoopDepth(MBB) > LI->getLoopDepth(SuccToSinkTo))
543  return true;
544 
545  // Check if only use in post dominated block is PHI instruction.
546  bool NonPHIUse = false;
547  for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg)) {
548  MachineBasicBlock *UseBlock = UseInst.getParent();
549  if (UseBlock == SuccToSinkTo && !UseInst.isPHI())
550  NonPHIUse = true;
551  }
552  if (!NonPHIUse)
553  return true;
554 
555  // If SuccToSinkTo post dominates then also it may be profitable if MI
556  // can further profitably sinked into another block in next round.
557  bool BreakPHIEdge = false;
558  // FIXME - If finding successor is compile time expensive then cache results.
559  if (MachineBasicBlock *MBB2 =
560  FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
561  return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
562 
563  // If SuccToSinkTo is final destination and it is a post dominator of current
564  // block then it is not profitable to sink MI into SuccToSinkTo block.
565  return false;
566 }
567 
568 /// Get the sorted sequence of successors for this MachineBasicBlock, possibly
569 /// computing it if it was not already cached.
571 MachineSinking::GetAllSortedSuccessors(MachineInstr &MI, MachineBasicBlock *MBB,
572  AllSuccsCache &AllSuccessors) const {
573 
574  // Do we have the sorted successors in cache ?
575  auto Succs = AllSuccessors.find(MBB);
576  if (Succs != AllSuccessors.end())
577  return Succs->second;
578 
580  MBB->succ_end());
581 
582  // Handle cases where sinking can happen but where the sink point isn't a
583  // successor. For example:
584  //
585  // x = computation
586  // if () {} else {}
587  // use x
588  //
589  const std::vector<MachineDomTreeNode *> &Children =
590  DT->getNode(MBB)->getChildren();
591  for (const auto &DTChild : Children)
592  // DomTree children of MBB that have MBB as immediate dominator are added.
593  if (DTChild->getIDom()->getBlock() == MI.getParent() &&
594  // Skip MBBs already added to the AllSuccs vector above.
595  !MBB->isSuccessor(DTChild->getBlock()))
596  AllSuccs.push_back(DTChild->getBlock());
597 
598  // Sort Successors according to their loop depth or block frequency info.
599  std::stable_sort(
600  AllSuccs.begin(), AllSuccs.end(),
601  [this](const MachineBasicBlock *L, const MachineBasicBlock *R) {
602  uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
603  uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
604  bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
605  return HasBlockFreq ? LHSFreq < RHSFreq
606  : LI->getLoopDepth(L) < LI->getLoopDepth(R);
607  });
608 
609  auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs));
610 
611  return it.first->second;
612 }
613 
614 /// FindSuccToSinkTo - Find a successor to sink this instruction to.
616 MachineSinking::FindSuccToSinkTo(MachineInstr &MI, MachineBasicBlock *MBB,
617  bool &BreakPHIEdge,
618  AllSuccsCache &AllSuccessors) {
619  assert (MBB && "Invalid MachineBasicBlock!");
620 
621  // Loop over all the operands of the specified instruction. If there is
622  // anything we can't handle, bail out.
623 
624  // SuccToSinkTo - This is the successor to sink this instruction to, once we
625  // decide.
626  MachineBasicBlock *SuccToSinkTo = nullptr;
627  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
628  const MachineOperand &MO = MI.getOperand(i);
629  if (!MO.isReg()) continue; // Ignore non-register operands.
630 
631  unsigned Reg = MO.getReg();
632  if (Reg == 0) continue;
633 
635  if (MO.isUse()) {
636  // If the physreg has no defs anywhere, it's just an ambient register
637  // and we can freely move its uses. Alternatively, if it's allocatable,
638  // it could get allocated to something with a def during allocation.
639  if (!MRI->isConstantPhysReg(Reg))
640  return nullptr;
641  } else if (!MO.isDead()) {
642  // A def that isn't dead. We can't move it.
643  return nullptr;
644  }
645  } else {
646  // Virtual register uses are always safe to sink.
647  if (MO.isUse()) continue;
648 
649  // If it's not safe to move defs of the register class, then abort.
650  if (!TII->isSafeToMoveRegClassDefs(MRI->getRegClass(Reg)))
651  return nullptr;
652 
653  // Virtual register defs can only be sunk if all their uses are in blocks
654  // dominated by one of the successors.
655  if (SuccToSinkTo) {
656  // If a previous operand picked a block to sink to, then this operand
657  // must be sinkable to the same block.
658  bool LocalUse = false;
659  if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, MBB,
660  BreakPHIEdge, LocalUse))
661  return nullptr;
662 
663  continue;
664  }
665 
666  // Otherwise, we should look at all the successors and decide which one
667  // we should sink to. If we have reliable block frequency information
668  // (frequency != 0) available, give successors with smaller frequencies
669  // higher priority, otherwise prioritize smaller loop depths.
670  for (MachineBasicBlock *SuccBlock :
671  GetAllSortedSuccessors(MI, MBB, AllSuccessors)) {
672  bool LocalUse = false;
673  if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB,
674  BreakPHIEdge, LocalUse)) {
675  SuccToSinkTo = SuccBlock;
676  break;
677  }
678  if (LocalUse)
679  // Def is used locally, it's never safe to move this def.
680  return nullptr;
681  }
682 
683  // If we couldn't find a block to sink to, ignore this instruction.
684  if (!SuccToSinkTo)
685  return nullptr;
686  if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo, AllSuccessors))
687  return nullptr;
688  }
689  }
690 
691  // It is not possible to sink an instruction into its own block. This can
692  // happen with loops.
693  if (MBB == SuccToSinkTo)
694  return nullptr;
695 
696  // It's not safe to sink instructions to EH landing pad. Control flow into
697  // landing pad is implicitly defined.
698  if (SuccToSinkTo && SuccToSinkTo->isEHPad())
699  return nullptr;
700 
701  return SuccToSinkTo;
702 }
703 
704 /// \brief Return true if MI is likely to be usable as a memory operation by the
705 /// implicit null check optimization.
706 ///
707 /// This is a "best effort" heuristic, and should not be relied upon for
708 /// correctness. This returning true does not guarantee that the implicit null
709 /// check optimization is legal over MI, and this returning false does not
710 /// guarantee MI cannot possibly be used to do a null check.
712  const TargetInstrInfo *TII,
713  const TargetRegisterInfo *TRI) {
714  typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate;
715 
716  auto *MBB = MI.getParent();
717  if (MBB->pred_size() != 1)
718  return false;
719 
720  auto *PredMBB = *MBB->pred_begin();
721  auto *PredBB = PredMBB->getBasicBlock();
722 
723  // Frontends that don't use implicit null checks have no reason to emit
724  // branches with make.implicit metadata, and this function should always
725  // return false for them.
726  if (!PredBB ||
727  !PredBB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit))
728  return false;
729 
730  unsigned BaseReg;
731  int64_t Offset;
732  if (!TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI))
733  return false;
734 
735  if (!(MI.mayLoad() && !MI.isPredicable()))
736  return false;
737 
738  MachineBranchPredicate MBP;
739  if (TII->analyzeBranchPredicate(*PredMBB, MBP, false))
740  return false;
741 
742  return MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
743  (MBP.Predicate == MachineBranchPredicate::PRED_NE ||
744  MBP.Predicate == MachineBranchPredicate::PRED_EQ) &&
745  MBP.LHS.getReg() == BaseReg;
746 }
747 
748 /// SinkInstruction - Determine whether it is safe to sink the specified machine
749 /// instruction out of its current block into a successor.
750 bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
751  AllSuccsCache &AllSuccessors) {
752  // Don't sink instructions that the target prefers not to sink.
753  if (!TII->shouldSink(MI))
754  return false;
755 
756  // Check if it's safe to move the instruction.
757  if (!MI.isSafeToMove(AA, SawStore))
758  return false;
759 
760  // Convergent operations may not be made control-dependent on additional
761  // values.
762  if (MI.isConvergent())
763  return false;
764 
765  // Don't break implicit null checks. This is a performance heuristic, and not
766  // required for correctness.
768  return false;
769 
770  // FIXME: This should include support for sinking instructions within the
771  // block they are currently in to shorten the live ranges. We often get
772  // instructions sunk into the top of a large block, but it would be better to
773  // also sink them down before their first use in the block. This xform has to
774  // be careful not to *increase* register pressure though, e.g. sinking
775  // "x = y + z" down if it kills y and z would increase the live ranges of y
776  // and z and only shrink the live range of x.
777 
778  bool BreakPHIEdge = false;
779  MachineBasicBlock *ParentBlock = MI.getParent();
780  MachineBasicBlock *SuccToSinkTo =
781  FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge, AllSuccessors);
782 
783  // If there are no outputs, it must have side-effects.
784  if (!SuccToSinkTo)
785  return false;
786 
787 
788  // If the instruction to move defines a dead physical register which is live
789  // when leaving the basic block, don't move it because it could turn into a
790  // "zombie" define of that preg. E.g., EFLAGS. (<rdar://problem/8030636>)
791  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
792  const MachineOperand &MO = MI.getOperand(I);
793  if (!MO.isReg()) continue;
794  unsigned Reg = MO.getReg();
795  if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
796  if (SuccToSinkTo->isLiveIn(Reg))
797  return false;
798  }
799 
800  DEBUG(dbgs() << "Sink instr " << MI << "\tinto block " << *SuccToSinkTo);
801 
802  // If the block has multiple predecessors, this is a critical edge.
803  // Decide if we can sink along it or need to break the edge.
804  if (SuccToSinkTo->pred_size() > 1) {
805  // We cannot sink a load across a critical edge - there may be stores in
806  // other code paths.
807  bool TryBreak = false;
808  bool store = true;
809  if (!MI.isSafeToMove(AA, store)) {
810  DEBUG(dbgs() << " *** NOTE: Won't sink load along critical edge.\n");
811  TryBreak = true;
812  }
813 
814  // We don't want to sink across a critical edge if we don't dominate the
815  // successor. We could be introducing calculations to new code paths.
816  if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
817  DEBUG(dbgs() << " *** NOTE: Critical edge found\n");
818  TryBreak = true;
819  }
820 
821  // Don't sink instructions into a loop.
822  if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
823  DEBUG(dbgs() << " *** NOTE: Loop header found\n");
824  TryBreak = true;
825  }
826 
827  // Otherwise we are OK with sinking along a critical edge.
828  if (!TryBreak)
829  DEBUG(dbgs() << "Sinking along critical edge.\n");
830  else {
831  // Mark this edge as to be split.
832  // If the edge can actually be split, the next iteration of the main loop
833  // will sink MI in the newly created block.
834  bool Status =
835  PostponeSplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
836  if (!Status)
837  DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
838  "break critical edge\n");
839  // The instruction will not be sunk this time.
840  return false;
841  }
842  }
843 
844  if (BreakPHIEdge) {
845  // BreakPHIEdge is true if all the uses are in the successor MBB being
846  // sunken into and they are all PHI nodes. In this case, machine-sink must
847  // break the critical edge first.
848  bool Status = PostponeSplitCriticalEdge(MI, ParentBlock,
849  SuccToSinkTo, BreakPHIEdge);
850  if (!Status)
851  DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
852  "break critical edge\n");
853  // The instruction will not be sunk this time.
854  return false;
855  }
856 
857  // Determine where to insert into. Skip phi nodes.
858  MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
859  while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
860  ++InsertPos;
861 
862  // collect matching debug values.
863  SmallVector<MachineInstr *, 2> DbgValuesToSink;
864  collectDebugValues(MI, DbgValuesToSink);
865 
866  // Move the instruction.
867  SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
869 
870  // Move debug values.
871  for (SmallVectorImpl<MachineInstr *>::iterator DBI = DbgValuesToSink.begin(),
872  DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) {
873  MachineInstr *DbgMI = *DBI;
874  SuccToSinkTo->splice(InsertPos, ParentBlock, DbgMI,
875  ++MachineBasicBlock::iterator(DbgMI));
876  }
877 
878  // Conservatively, clear any kill flags, since it's possible that they are no
879  // longer correct.
880  // Note that we have to clear the kill flags for any register this instruction
881  // uses as we may sink over another instruction which currently kills the
882  // used registers.
883  for (MachineOperand &MO : MI.operands()) {
884  if (MO.isReg() && MO.isUse())
885  RegsToClearKillFlags.set(MO.getReg()); // Remember to clear kill flags.
886  }
887 
888  return true;
889 }
MachineLoop * L
unsigned succ_size() const
bool isEHPad() const
Returns true if the block is a landing pad.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static bool AllUsesDominatedByBlock(Instruction *Inst, BasicBlock *BB, DominatorTree &DT)
AllUsesDominatedByBlock - Return true if all uses of the specified value occur in blocks dominated by...
Definition: Sink.cpp:37
STATISTIC(NumFunctions,"Total number of functions")
size_t i
MachineBasicBlock * getMBB() const
bool isDead() const
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
virtual bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify=false) const
Analyze the branching code at the end of MBB and parse it into the MachineBranchPredicate structure i...
bool isPredicable(QueryType Type=AllInBundle) const
Return true if this instruction has a predicate operand that controls execution.
Definition: MachineInstr.h:478
machine Machine code sinking
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:301
Represents a predicate at the MachineFunction level.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
bool isPHI() const
Definition: MachineInstr.h:786
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:592
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:277
MachineBasicBlock * MBB
bool isCopyLike() const
Return true if the instruction behaves like a copy.
Definition: MachineInstr.h:819
std::vector< MachineBasicBlock * >::iterator pred_iterator
COFF::MachineTypes Machine
Definition: COFFYAML.cpp:303
static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
Definition: Sink.cpp:200
bool isConvergent(QueryType Type=AnyInBundle) const
Return true if this instruction is convergent.
Definition: MachineInstr.h:515
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:131
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:777
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:279
bool isCopy() const
Definition: MachineInstr.h:807
Represent the analysis usage information of a pass.
uint32_t Offset
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
Return true if MI is likely to be usable as a memory operation by the implicit null check optimizatio...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
machine Machine code false
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
static bool SinkInstruction(Instruction *Inst, SmallPtrSetImpl< Instruction * > &Stores, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
SinkInstruction - Determine whether it is safe to sink the specified machine instruction out of its c...
Definition: Sink.cpp:139
static cl::opt< unsigned > SplitEdgeProbabilityThreshold("machine-sink-split-probability-threshold", cl::desc("Percentage threshold for splitting single-instruction critical edge. ""If the branch threshold is higher than this threshold, we allow ""speculative execution of up to 1 instruction to avoid branching to ""splitted critical edge"), cl::init(40), cl::Hidden)
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:276
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
virtual bool getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, int64_t &Offset, const TargetRegisterInfo *TRI) const
Get the base register and byte offset of an instruction that reads/writes memory. ...
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:52
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
#define I(x, y, z)
Definition: MD5.cpp:54
void initializeMachineSinkingPass(PassRegistry &)
INITIALIZE_PASS_BEGIN(MachineSinking,"machine-sink","Machine code sinking", false, false) INITIALIZE_PASS_END(MachineSinking
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
static void collectDebugValues(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DbgValues)
collectDebgValues - Scan instructions following MI and collect any matching DBG_VALUEs.
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
virtual const TargetInstrInfo * getInstrInfo() const
A vector that has set insertion semantics.
Definition: SetVector.h:41
#define DEBUG(X)
Definition: Debug.h:100
IRTranslator LLVM IR MI
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
machine sink
When an instruction is found to only be used outside of the loop, this function moves it to the exit ...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const