LLVM  3.7.0
MachineSink.cpp
Go to the documentation of this file.
1 //===-- MachineSink.cpp - Sinking for machine instructions ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass moves instructions into successor blocks when possible, so that
11 // they aren't executed on paths where their results aren't needed.
12 //
13 // This pass is not intended to be a replacement or a complete alternative
14 // for an LLVM-IR-level sinking pass. It is only designed to sink simple
15 // constructs that are not exposed before lowering and instruction selection.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "llvm/CodeGen/Passes.h"
20 #include "llvm/ADT/SetVector.h"
21 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/Statistic.h"
31 #include "llvm/Support/Debug.h"
36 using namespace llvm;
37 
38 #define DEBUG_TYPE "machine-sink"
39 
40 static cl::opt<bool>
41 SplitEdges("machine-sink-split",
42  cl::desc("Split critical edges during machine sinking"),
43  cl::init(true), cl::Hidden);
44 
45 static cl::opt<bool>
46 UseBlockFreqInfo("machine-sink-bfi",
47  cl::desc("Use block frequency info to find successors to sink"),
48  cl::init(true), cl::Hidden);
49 
50 
51 STATISTIC(NumSunk, "Number of machine instructions sunk");
52 STATISTIC(NumSplit, "Number of critical edges split");
53 STATISTIC(NumCoalesces, "Number of copies coalesced");
54 
55 namespace {
56  class MachineSinking : public MachineFunctionPass {
57  const TargetInstrInfo *TII;
58  const TargetRegisterInfo *TRI;
59  MachineRegisterInfo *MRI; // Machine register information
60  MachineDominatorTree *DT; // Machine dominator tree
61  MachinePostDominatorTree *PDT; // Machine post dominator tree
62  MachineLoopInfo *LI;
63  const MachineBlockFrequencyInfo *MBFI;
64  AliasAnalysis *AA;
65 
66  // Remember which edges have been considered for breaking.
68  CEBCandidates;
69  // Remember which edges we are about to split.
70  // This is different from CEBCandidates since those edges
71  // will be split.
73 
74  SparseBitVector<> RegsToClearKillFlags;
75 
76  typedef std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>
77  AllSuccsCache;
78 
79  public:
80  static char ID; // Pass identification
81  MachineSinking() : MachineFunctionPass(ID) {
83  }
84 
85  bool runOnMachineFunction(MachineFunction &MF) override;
86 
87  void getAnalysisUsage(AnalysisUsage &AU) const override {
88  AU.setPreservesCFG();
97  if (UseBlockFreqInfo)
99  }
100 
101  void releaseMemory() override {
102  CEBCandidates.clear();
103  }
104 
105  private:
106  bool ProcessBlock(MachineBasicBlock &MBB);
107  bool isWorthBreakingCriticalEdge(MachineInstr *MI,
108  MachineBasicBlock *From,
109  MachineBasicBlock *To);
110  /// \brief Postpone the splitting of the given critical
111  /// edge (\p From, \p To).
112  ///
113  /// We do not split the edges on the fly. Indeed, this invalidates
114  /// the dominance information and thus triggers a lot of updates
115  /// of that information underneath.
116  /// Instead, we postpone all the splits after each iteration of
117  /// the main loop. That way, the information is at least valid
118  /// for the lifetime of an iteration.
119  ///
120  /// \return True if the edge is marked as toSplit, false otherwise.
121  /// False can be returned if, for instance, this is not profitable.
122  bool PostponeSplitCriticalEdge(MachineInstr *MI,
123  MachineBasicBlock *From,
124  MachineBasicBlock *To,
125  bool BreakPHIEdge);
126  bool SinkInstruction(MachineInstr *MI, bool &SawStore,
127  AllSuccsCache &AllSuccessors);
128  bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB,
129  MachineBasicBlock *DefMBB,
130  bool &BreakPHIEdge, bool &LocalUse) const;
131  MachineBasicBlock *FindSuccToSinkTo(MachineInstr *MI, MachineBasicBlock *MBB,
132  bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
133  bool isProfitableToSinkTo(unsigned Reg, MachineInstr *MI,
134  MachineBasicBlock *MBB,
135  MachineBasicBlock *SuccToSinkTo,
136  AllSuccsCache &AllSuccessors);
137 
138  bool PerformTrivialForwardCoalescing(MachineInstr *MI,
139  MachineBasicBlock *MBB);
140 
142  GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB,
143  AllSuccsCache &AllSuccessors) const;
144  };
145 } // end anonymous namespace
146 
147 char MachineSinking::ID = 0;
149 INITIALIZE_PASS_BEGIN(MachineSinking, "machine-sink",
150  "Machine code sinking", false, false)
154 INITIALIZE_PASS_END(MachineSinking, "machine-sink",
155  "Machine code sinking", false, false)
156 
157 bool MachineSinking::PerformTrivialForwardCoalescing(MachineInstr *MI,
158  MachineBasicBlock *MBB) {
159  if (!MI->isCopy())
160  return false;
161 
162  unsigned SrcReg = MI->getOperand(1).getReg();
163  unsigned DstReg = MI->getOperand(0).getReg();
166  !MRI->hasOneNonDBGUse(SrcReg))
167  return false;
168 
169  const TargetRegisterClass *SRC = MRI->getRegClass(SrcReg);
170  const TargetRegisterClass *DRC = MRI->getRegClass(DstReg);
171  if (SRC != DRC)
172  return false;
173 
174  MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
175  if (DefMI->isCopyLike())
176  return false;
177  DEBUG(dbgs() << "Coalescing: " << *DefMI);
178  DEBUG(dbgs() << "*** to: " << *MI);
179  MRI->replaceRegWith(DstReg, SrcReg);
180  MI->eraseFromParent();
181 
182  // Conservatively, clear any kill flags, since it's possible that they are no
183  // longer correct.
184  MRI->clearKillFlags(SrcReg);
185 
186  ++NumCoalesces;
187  return true;
188 }
189 
190 /// AllUsesDominatedByBlock - Return true if all uses of the specified register
191 /// occur in blocks dominated by the specified block. If any use is in the
192 /// definition block, then return false since it is never legal to move def
193 /// after uses.
194 bool
195 MachineSinking::AllUsesDominatedByBlock(unsigned Reg,
196  MachineBasicBlock *MBB,
197  MachineBasicBlock *DefMBB,
198  bool &BreakPHIEdge,
199  bool &LocalUse) const {
201  "Only makes sense for vregs");
202 
203  // Ignore debug uses because debug info doesn't affect the code.
204  if (MRI->use_nodbg_empty(Reg))
205  return true;
206 
207  // BreakPHIEdge is true if all the uses are in the successor MBB being sunken
208  // into and they are all PHI nodes. In this case, machine-sink must break
209  // the critical edge first. e.g.
210  //
211  // BB#1: derived from LLVM BB %bb4.preheader
212  // Predecessors according to CFG: BB#0
213  // ...
214  // %reg16385<def> = DEC64_32r %reg16437, %EFLAGS<imp-def,dead>
215  // ...
216  // JE_4 <BB#37>, %EFLAGS<imp-use>
217  // Successors according to CFG: BB#37 BB#2
218  //
219  // BB#2: derived from LLVM BB %bb.nph
220  // Predecessors according to CFG: BB#0 BB#1
221  // %reg16386<def> = PHI %reg16434, <BB#0>, %reg16385, <BB#1>
222  BreakPHIEdge = true;
223  for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
224  MachineInstr *UseInst = MO.getParent();
225  unsigned OpNo = &MO - &UseInst->getOperand(0);
226  MachineBasicBlock *UseBlock = UseInst->getParent();
227  if (!(UseBlock == MBB && UseInst->isPHI() &&
228  UseInst->getOperand(OpNo+1).getMBB() == DefMBB)) {
229  BreakPHIEdge = false;
230  break;
231  }
232  }
233  if (BreakPHIEdge)
234  return true;
235 
236  for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
237  // Determine the block of the use.
238  MachineInstr *UseInst = MO.getParent();
239  unsigned OpNo = &MO - &UseInst->getOperand(0);
240  MachineBasicBlock *UseBlock = UseInst->getParent();
241  if (UseInst->isPHI()) {
242  // PHI nodes use the operand in the predecessor block, not the block with
243  // the PHI.
244  UseBlock = UseInst->getOperand(OpNo+1).getMBB();
245  } else if (UseBlock == DefMBB) {
246  LocalUse = true;
247  return false;
248  }
249 
250  // Check that it dominates.
251  if (!DT->dominates(MBB, UseBlock))
252  return false;
253  }
254 
255  return true;
256 }
257 
258 bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
259  if (skipOptnoneFunction(*MF.getFunction()))
260  return false;
261 
262  DEBUG(dbgs() << "******** Machine Sinking ********\n");
263 
264  TII = MF.getSubtarget().getInstrInfo();
265  TRI = MF.getSubtarget().getRegisterInfo();
266  MRI = &MF.getRegInfo();
267  DT = &getAnalysis<MachineDominatorTree>();
268  PDT = &getAnalysis<MachinePostDominatorTree>();
269  LI = &getAnalysis<MachineLoopInfo>();
270  MBFI = UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() : nullptr;
271  AA = &getAnalysis<AliasAnalysis>();
272 
273  bool EverMadeChange = false;
274 
275  while (1) {
276  bool MadeChange = false;
277 
278  // Process all basic blocks.
279  CEBCandidates.clear();
280  ToSplit.clear();
281  for (auto &MBB: MF)
282  MadeChange |= ProcessBlock(MBB);
283 
284  // If we have anything we marked as toSplit, split it now.
285  for (auto &Pair : ToSplit) {
286  auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, this);
287  if (NewSucc != nullptr) {
288  DEBUG(dbgs() << " *** Splitting critical edge:"
289  " BB#" << Pair.first->getNumber()
290  << " -- BB#" << NewSucc->getNumber()
291  << " -- BB#" << Pair.second->getNumber() << '\n');
292  MadeChange = true;
293  ++NumSplit;
294  } else
295  DEBUG(dbgs() << " *** Not legal to break critical edge\n");
296  }
297  // If this iteration over the code changed anything, keep iterating.
298  if (!MadeChange) break;
299  EverMadeChange = true;
300  }
301 
302  // Now clear any kill flags for recorded registers.
303  for (auto I : RegsToClearKillFlags)
304  MRI->clearKillFlags(I);
305  RegsToClearKillFlags.clear();
306 
307  return EverMadeChange;
308 }
309 
310 bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
311  // Can't sink anything out of a block that has less than two successors.
312  if (MBB.succ_size() <= 1 || MBB.empty()) return false;
313 
314  // Don't bother sinking code out of unreachable blocks. In addition to being
315  // unprofitable, it can also lead to infinite looping, because in an
316  // unreachable loop there may be nowhere to stop.
317  if (!DT->isReachableFromEntry(&MBB)) return false;
318 
319  bool MadeChange = false;
320 
321  // Cache all successors, sorted by frequency info and loop depth.
322  AllSuccsCache AllSuccessors;
323 
324  // Walk the basic block bottom-up. Remember if we saw a store.
326  --I;
327  bool ProcessedBegin, SawStore = false;
328  do {
329  MachineInstr *MI = I; // The instruction to sink.
330 
331  // Predecrement I (if it's not begin) so that it isn't invalidated by
332  // sinking.
333  ProcessedBegin = I == MBB.begin();
334  if (!ProcessedBegin)
335  --I;
336 
337  if (MI->isDebugValue())
338  continue;
339 
340  bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
341  if (Joined) {
342  MadeChange = true;
343  continue;
344  }
345 
346  if (SinkInstruction(MI, SawStore, AllSuccessors))
347  ++NumSunk, MadeChange = true;
348 
349  // If we just processed the first instruction in the block, we're done.
350  } while (!ProcessedBegin);
351 
352  return MadeChange;
353 }
354 
355 bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr *MI,
356  MachineBasicBlock *From,
357  MachineBasicBlock *To) {
358  // FIXME: Need much better heuristics.
359 
360  // If the pass has already considered breaking this edge (during this pass
361  // through the function), then let's go ahead and break it. This means
362  // sinking multiple "cheap" instructions into the same block.
363  if (!CEBCandidates.insert(std::make_pair(From, To)).second)
364  return true;
365 
366  if (!MI->isCopy() && !TII->isAsCheapAsAMove(MI))
367  return true;
368 
369  // MI is cheap, we probably don't want to break the critical edge for it.
370  // However, if this would allow some definitions of its source operands
371  // to be sunk then it's probably worth it.
372  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
373  const MachineOperand &MO = MI->getOperand(i);
374  if (!MO.isReg() || !MO.isUse())
375  continue;
376  unsigned Reg = MO.getReg();
377  if (Reg == 0)
378  continue;
379 
380  // We don't move live definitions of physical registers,
381  // so sinking their uses won't enable any opportunities.
383  continue;
384 
385  // If this instruction is the only user of a virtual register,
386  // check if breaking the edge will enable sinking
387  // both this instruction and the defining instruction.
388  if (MRI->hasOneNonDBGUse(Reg)) {
389  // If the definition resides in same MBB,
390  // claim it's likely we can sink these together.
391  // If definition resides elsewhere, we aren't
392  // blocking it from being sunk so don't break the edge.
393  MachineInstr *DefMI = MRI->getVRegDef(Reg);
394  if (DefMI->getParent() == MI->getParent())
395  return true;
396  }
397  }
398 
399  return false;
400 }
401 
402 bool MachineSinking::PostponeSplitCriticalEdge(MachineInstr *MI,
403  MachineBasicBlock *FromBB,
404  MachineBasicBlock *ToBB,
405  bool BreakPHIEdge) {
406  if (!isWorthBreakingCriticalEdge(MI, FromBB, ToBB))
407  return false;
408 
409  // Avoid breaking back edge. From == To means backedge for single BB loop.
410  if (!SplitEdges || FromBB == ToBB)
411  return false;
412 
413  // Check for backedges of more "complex" loops.
414  if (LI->getLoopFor(FromBB) == LI->getLoopFor(ToBB) &&
415  LI->isLoopHeader(ToBB))
416  return false;
417 
418  // It's not always legal to break critical edges and sink the computation
419  // to the edge.
420  //
421  // BB#1:
422  // v1024
423  // Beq BB#3
424  // <fallthrough>
425  // BB#2:
426  // ... no uses of v1024
427  // <fallthrough>
428  // BB#3:
429  // ...
430  // = v1024
431  //
432  // If BB#1 -> BB#3 edge is broken and computation of v1024 is inserted:
433  //
434  // BB#1:
435  // ...
436  // Bne BB#2
437  // BB#4:
438  // v1024 =
439  // B BB#3
440  // BB#2:
441  // ... no uses of v1024
442  // <fallthrough>
443  // BB#3:
444  // ...
445  // = v1024
446  //
447  // This is incorrect since v1024 is not computed along the BB#1->BB#2->BB#3
448  // flow. We need to ensure the new basic block where the computation is
449  // sunk to dominates all the uses.
450  // It's only legal to break critical edge and sink the computation to the
451  // new block if all the predecessors of "To", except for "From", are
452  // not dominated by "From". Given SSA property, this means these
453  // predecessors are dominated by "To".
454  //
455  // There is no need to do this check if all the uses are PHI nodes. PHI
456  // sources are only defined on the specific predecessor edges.
457  if (!BreakPHIEdge) {
459  E = ToBB->pred_end(); PI != E; ++PI) {
460  if (*PI == FromBB)
461  continue;
462  if (!DT->dominates(ToBB, *PI))
463  return false;
464  }
465  }
466 
467  ToSplit.insert(std::make_pair(FromBB, ToBB));
468 
469  return true;
470 }
471 
473  return MI->isInsertSubreg() || MI->isSubregToReg() || MI->isRegSequence();
474 }
475 
476 /// collectDebgValues - Scan instructions following MI and collect any
477 /// matching DBG_VALUEs.
479  SmallVectorImpl<MachineInstr *> &DbgValues) {
480  DbgValues.clear();
481  if (!MI->getOperand(0).isReg())
482  return;
483 
485  for (MachineBasicBlock::iterator DE = MI->getParent()->end();
486  DI != DE; ++DI) {
487  if (!DI->isDebugValue())
488  return;
489  if (DI->getOperand(0).isReg() &&
490  DI->getOperand(0).getReg() == MI->getOperand(0).getReg())
491  DbgValues.push_back(DI);
492  }
493 }
494 
495 /// isProfitableToSinkTo - Return true if it is profitable to sink MI.
496 bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr *MI,
497  MachineBasicBlock *MBB,
498  MachineBasicBlock *SuccToSinkTo,
499  AllSuccsCache &AllSuccessors) {
500  assert (MI && "Invalid MachineInstr!");
501  assert (SuccToSinkTo && "Invalid SinkTo Candidate BB");
502 
503  if (MBB == SuccToSinkTo)
504  return false;
505 
506  // It is profitable if SuccToSinkTo does not post dominate current block.
507  if (!PDT->dominates(SuccToSinkTo, MBB))
508  return true;
509 
510  // It is profitable to sink an instruction from a deeper loop to a shallower
511  // loop, even if the latter post-dominates the former (PR21115).
512  if (LI->getLoopDepth(MBB) > LI->getLoopDepth(SuccToSinkTo))
513  return true;
514 
515  // Check if only use in post dominated block is PHI instruction.
516  bool NonPHIUse = false;
517  for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg)) {
518  MachineBasicBlock *UseBlock = UseInst.getParent();
519  if (UseBlock == SuccToSinkTo && !UseInst.isPHI())
520  NonPHIUse = true;
521  }
522  if (!NonPHIUse)
523  return true;
524 
525  // If SuccToSinkTo post dominates then also it may be profitable if MI
526  // can further profitably sinked into another block in next round.
527  bool BreakPHIEdge = false;
528  // FIXME - If finding successor is compile time expensive then cache results.
529  if (MachineBasicBlock *MBB2 =
530  FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
531  return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
532 
533  // If SuccToSinkTo is final destination and it is a post dominator of current
534  // block then it is not profitable to sink MI into SuccToSinkTo block.
535  return false;
536 }
537 
538 /// Get the sorted sequence of successors for this MachineBasicBlock, possibly
539 /// computing it if it was not already cached.
541 MachineSinking::GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB,
542  AllSuccsCache &AllSuccessors) const {
543 
544  // Do we have the sorted successors in cache ?
545  auto Succs = AllSuccessors.find(MBB);
546  if (Succs != AllSuccessors.end())
547  return Succs->second;
548 
550  MBB->succ_end());
551 
552  // Handle cases where sinking can happen but where the sink point isn't a
553  // successor. For example:
554  //
555  // x = computation
556  // if () {} else {}
557  // use x
558  //
559  const std::vector<MachineDomTreeNode *> &Children =
560  DT->getNode(MBB)->getChildren();
561  for (const auto &DTChild : Children)
562  // DomTree children of MBB that have MBB as immediate dominator are added.
563  if (DTChild->getIDom()->getBlock() == MI->getParent() &&
564  // Skip MBBs already added to the AllSuccs vector above.
565  !MBB->isSuccessor(DTChild->getBlock()))
566  AllSuccs.push_back(DTChild->getBlock());
567 
568  // Sort Successors according to their loop depth or block frequency info.
569  std::stable_sort(
570  AllSuccs.begin(), AllSuccs.end(),
571  [this](const MachineBasicBlock *L, const MachineBasicBlock *R) {
572  uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
573  uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
574  bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
575  return HasBlockFreq ? LHSFreq < RHSFreq
576  : LI->getLoopDepth(L) < LI->getLoopDepth(R);
577  });
578 
579  auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs));
580 
581  return it.first->second;
582 }
583 
584 /// FindSuccToSinkTo - Find a successor to sink this instruction to.
585 MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI,
586  MachineBasicBlock *MBB,
587  bool &BreakPHIEdge,
588  AllSuccsCache &AllSuccessors) {
589 
590  assert (MI && "Invalid MachineInstr!");
591  assert (MBB && "Invalid MachineBasicBlock!");
592 
593  // Loop over all the operands of the specified instruction. If there is
594  // anything we can't handle, bail out.
595 
596  // SuccToSinkTo - This is the successor to sink this instruction to, once we
597  // decide.
598  MachineBasicBlock *SuccToSinkTo = nullptr;
599  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
600  const MachineOperand &MO = MI->getOperand(i);
601  if (!MO.isReg()) continue; // Ignore non-register operands.
602 
603  unsigned Reg = MO.getReg();
604  if (Reg == 0) continue;
605 
607  if (MO.isUse()) {
608  // If the physreg has no defs anywhere, it's just an ambient register
609  // and we can freely move its uses. Alternatively, if it's allocatable,
610  // it could get allocated to something with a def during allocation.
611  if (!MRI->isConstantPhysReg(Reg, *MBB->getParent()))
612  return nullptr;
613  } else if (!MO.isDead()) {
614  // A def that isn't dead. We can't move it.
615  return nullptr;
616  }
617  } else {
618  // Virtual register uses are always safe to sink.
619  if (MO.isUse()) continue;
620 
621  // If it's not safe to move defs of the register class, then abort.
622  if (!TII->isSafeToMoveRegClassDefs(MRI->getRegClass(Reg)))
623  return nullptr;
624 
625  // Virtual register defs can only be sunk if all their uses are in blocks
626  // dominated by one of the successors.
627  if (SuccToSinkTo) {
628  // If a previous operand picked a block to sink to, then this operand
629  // must be sinkable to the same block.
630  bool LocalUse = false;
631  if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, MBB,
632  BreakPHIEdge, LocalUse))
633  return nullptr;
634 
635  continue;
636  }
637 
638  // Otherwise, we should look at all the successors and decide which one
639  // we should sink to. If we have reliable block frequency information
640  // (frequency != 0) available, give successors with smaller frequencies
641  // higher priority, otherwise prioritize smaller loop depths.
642  for (MachineBasicBlock *SuccBlock :
643  GetAllSortedSuccessors(MI, MBB, AllSuccessors)) {
644  bool LocalUse = false;
645  if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB,
646  BreakPHIEdge, LocalUse)) {
647  SuccToSinkTo = SuccBlock;
648  break;
649  }
650  if (LocalUse)
651  // Def is used locally, it's never safe to move this def.
652  return nullptr;
653  }
654 
655  // If we couldn't find a block to sink to, ignore this instruction.
656  if (!SuccToSinkTo)
657  return nullptr;
658  if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo, AllSuccessors))
659  return nullptr;
660  }
661  }
662 
663  // It is not possible to sink an instruction into its own block. This can
664  // happen with loops.
665  if (MBB == SuccToSinkTo)
666  return nullptr;
667 
668  // It's not safe to sink instructions to EH landing pad. Control flow into
669  // landing pad is implicitly defined.
670  if (SuccToSinkTo && SuccToSinkTo->isLandingPad())
671  return nullptr;
672 
673  return SuccToSinkTo;
674 }
675 
676 /// SinkInstruction - Determine whether it is safe to sink the specified machine
677 /// instruction out of its current block into a successor.
678 bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore,
679  AllSuccsCache &AllSuccessors) {
680  // Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to
681  // be close to the source to make it easier to coalesce.
682  if (AvoidsSinking(MI, MRI))
683  return false;
684 
685  // Check if it's safe to move the instruction.
686  if (!MI->isSafeToMove(AA, SawStore))
687  return false;
688 
689  // Convergent operations may only be moved to control equivalent locations.
690  if (MI->isConvergent())
691  return false;
692 
693  // FIXME: This should include support for sinking instructions within the
694  // block they are currently in to shorten the live ranges. We often get
695  // instructions sunk into the top of a large block, but it would be better to
696  // also sink them down before their first use in the block. This xform has to
697  // be careful not to *increase* register pressure though, e.g. sinking
698  // "x = y + z" down if it kills y and z would increase the live ranges of y
699  // and z and only shrink the live range of x.
700 
701  bool BreakPHIEdge = false;
702  MachineBasicBlock *ParentBlock = MI->getParent();
703  MachineBasicBlock *SuccToSinkTo =
704  FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge, AllSuccessors);
705 
706  // If there are no outputs, it must have side-effects.
707  if (!SuccToSinkTo)
708  return false;
709 
710 
711  // If the instruction to move defines a dead physical register which is live
712  // when leaving the basic block, don't move it because it could turn into a
713  // "zombie" define of that preg. E.g., EFLAGS. (<rdar://problem/8030636>)
714  for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
715  const MachineOperand &MO = MI->getOperand(I);
716  if (!MO.isReg()) continue;
717  unsigned Reg = MO.getReg();
718  if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
719  if (SuccToSinkTo->isLiveIn(Reg))
720  return false;
721  }
722 
723  DEBUG(dbgs() << "Sink instr " << *MI << "\tinto block " << *SuccToSinkTo);
724 
725  // If the block has multiple predecessors, this is a critical edge.
726  // Decide if we can sink along it or need to break the edge.
727  if (SuccToSinkTo->pred_size() > 1) {
728  // We cannot sink a load across a critical edge - there may be stores in
729  // other code paths.
730  bool TryBreak = false;
731  bool store = true;
732  if (!MI->isSafeToMove(AA, store)) {
733  DEBUG(dbgs() << " *** NOTE: Won't sink load along critical edge.\n");
734  TryBreak = true;
735  }
736 
737  // We don't want to sink across a critical edge if we don't dominate the
738  // successor. We could be introducing calculations to new code paths.
739  if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
740  DEBUG(dbgs() << " *** NOTE: Critical edge found\n");
741  TryBreak = true;
742  }
743 
744  // Don't sink instructions into a loop.
745  if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
746  DEBUG(dbgs() << " *** NOTE: Loop header found\n");
747  TryBreak = true;
748  }
749 
750  // Otherwise we are OK with sinking along a critical edge.
751  if (!TryBreak)
752  DEBUG(dbgs() << "Sinking along critical edge.\n");
753  else {
754  // Mark this edge as to be split.
755  // If the edge can actually be split, the next iteration of the main loop
756  // will sink MI in the newly created block.
757  bool Status =
758  PostponeSplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
759  if (!Status)
760  DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
761  "break critical edge\n");
762  // The instruction will not be sunk this time.
763  return false;
764  }
765  }
766 
767  if (BreakPHIEdge) {
768  // BreakPHIEdge is true if all the uses are in the successor MBB being
769  // sunken into and they are all PHI nodes. In this case, machine-sink must
770  // break the critical edge first.
771  bool Status = PostponeSplitCriticalEdge(MI, ParentBlock,
772  SuccToSinkTo, BreakPHIEdge);
773  if (!Status)
774  DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
775  "break critical edge\n");
776  // The instruction will not be sunk this time.
777  return false;
778  }
779 
780  // Determine where to insert into. Skip phi nodes.
781  MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
782  while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
783  ++InsertPos;
784 
785  // collect matching debug values.
786  SmallVector<MachineInstr *, 2> DbgValuesToSink;
787  collectDebugValues(MI, DbgValuesToSink);
788 
789  // Move the instruction.
790  SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
792 
793  // Move debug values.
794  for (SmallVectorImpl<MachineInstr *>::iterator DBI = DbgValuesToSink.begin(),
795  DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) {
796  MachineInstr *DbgMI = *DBI;
797  SuccToSinkTo->splice(InsertPos, ParentBlock, DbgMI,
798  ++MachineBasicBlock::iterator(DbgMI));
799  }
800 
801  // Conservatively, clear any kill flags, since it's possible that they are no
802  // longer correct.
803  // Note that we have to clear the kill flags for any register this instruction
804  // uses as we may sink over another instruction which currently kills the
805  // used registers.
806  for (MachineOperand &MO : MI->operands()) {
807  if (MO.isReg() && MO.isUse())
808  RegsToClearKillFlags.set(MO.getReg()); // Remember to clear kill flags.
809  }
810 
811  return true;
812 }
unsigned succ_size() const
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
MachineBasicBlock * getMBB() const
bool isDead() const
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
machine Machine code sinking
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:295
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
bool isPHI() const
Definition: MachineInstr.h:757
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
bool isCopyLike() const
Return true if the instruction behaves like a copy.
Definition: MachineInstr.h:790
std::vector< MachineBasicBlock * >::iterator pred_iterator
COFF::MachineTypes Machine
Definition: COFFYAML.cpp:301
bool isConvergent(QueryType Type=AnyInBundle) const
Return true if this instruction is convergent.
Definition: MachineInstr.h:494
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:748
bool isInsertSubreg() const
Definition: MachineInstr.h:766
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
static bool AvoidsSinking(MachineInstr *MI, MachineRegisterInfo *MRI)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
bool isCopy() const
Definition: MachineInstr.h:778
Represent the analysis usage information of a pass.
#define INITIALIZE_AG_DEPENDENCY(depName)
Definition: PassSupport.h:72
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
machine Machine code false
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
static void collectDebugValues(MachineInstr *MI, SmallVectorImpl< MachineInstr * > &DbgValues)
collectDebgValues - Scan instructions following MI and collect any matching DBG_VALUEs.
bool isSubregToReg() const
Definition: MachineInstr.h:769
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:51
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
bool isLandingPad() const
isLandingPad - Returns true if the block is a landing pad.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
bool isLiveIn(unsigned Reg) const
isLiveIn - Return true if the specified register is in the live in set.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
#define I(x, y, z)
Definition: MD5.cpp:54
void initializeMachineSinkingPass(PassRegistry &)
INITIALIZE_PASS_BEGIN(MachineSinking,"machine-sink","Machine code sinking", false, false) INITIALIZE_PASS_END(MachineSinking
unsigned getReg() const
getReg - Returns the register number.
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
virtual const TargetInstrInfo * getInstrInfo() const
A vector that has set insertion semantics.
Definition: SetVector.h:37
#define DEBUG(X)
Definition: Debug.h:92
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool isRegSequence() const
Definition: MachineInstr.h:772
machine sink
When an instruction is found to only be used outside of the loop, this function moves it to the exit ...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
unsigned pred_size() const