LLVM  4.0.0
RegAllocGreedy.cpp
Go to the documentation of this file.
1 //===-- RegAllocGreedy.cpp - greedy register allocator --------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the RAGreedy function pass for register allocation in
11 // optimized builds.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AllocationOrder.h"
16 #include "InterferenceCache.h"
17 #include "LiveDebugVariables.h"
18 #include "RegAllocBase.h"
19 #include "SpillPlacement.h"
20 #include "Spiller.h"
21 #include "SplitKit.h"
22 #include "llvm/ADT/Statistic.h"
35 #include "llvm/CodeGen/Passes.h"
39 #include "llvm/IR/LLVMContext.h"
43 #include "llvm/Support/Debug.h"
45 #include "llvm/Support/Timer.h"
49 #include <queue>
50 
51 using namespace llvm;
52 
53 #define DEBUG_TYPE "regalloc"
54 
55 STATISTIC(NumGlobalSplits, "Number of split global live ranges");
56 STATISTIC(NumLocalSplits, "Number of split local live ranges");
57 STATISTIC(NumEvicted, "Number of interferences evicted");
58 
60  "split-spill-mode", cl::Hidden,
61  cl::desc("Spill mode for splitting live ranges"),
62  cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"),
63  clEnumValN(SplitEditor::SM_Size, "size", "Optimize for size"),
64  clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed")),
66 
67 static cl::opt<unsigned>
69  cl::desc("Last chance recoloring max depth"),
70  cl::init(5));
71 
73  "lcr-max-interf", cl::Hidden,
74  cl::desc("Last chance recoloring maximum number of considered"
75  " interference at a time"),
76  cl::init(8));
77 
78 static cl::opt<bool>
79 ExhaustiveSearch("exhaustive-register-search", cl::NotHidden,
80  cl::desc("Exhaustive Search for registers bypassing the depth "
81  "and interference cutoffs of last chance recoloring"));
82 
84  "enable-local-reassign", cl::Hidden,
85  cl::desc("Local reassignment can yield better allocation decisions, but "
86  "may be compile time intensive"),
87  cl::init(false));
88 
90  "enable-deferred-spilling", cl::Hidden,
91  cl::desc("Instead of spilling a variable right away, defer the actual "
92  "code insertion to the end of the allocation. That way the "
93  "allocator might still find a suitable coloring for this "
94  "variable because of other evicted variables."),
95  cl::init(false));
96 
97 // FIXME: Find a good default for this flag and remove the flag.
98 static cl::opt<unsigned>
99 CSRFirstTimeCost("regalloc-csr-first-time-cost",
100  cl::desc("Cost for first time use of callee-saved register."),
101  cl::init(0), cl::Hidden);
102 
103 static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator",
105 
106 namespace {
107 class RAGreedy : public MachineFunctionPass,
108  public RegAllocBase,
109  private LiveRangeEdit::Delegate {
110  // Convenient shortcuts.
111  typedef std::priority_queue<std::pair<unsigned, unsigned> > PQueue;
112  typedef SmallPtrSet<LiveInterval *, 4> SmallLISet;
113  typedef SmallSet<unsigned, 16> SmallVirtRegSet;
114 
115  // context
116  MachineFunction *MF;
117 
118  // Shortcuts to some useful interface.
119  const TargetInstrInfo *TII;
120  const TargetRegisterInfo *TRI;
121  RegisterClassInfo RCI;
122 
123  // analyses
124  SlotIndexes *Indexes;
126  MachineDominatorTree *DomTree;
128  EdgeBundles *Bundles;
129  SpillPlacement *SpillPlacer;
130  LiveDebugVariables *DebugVars;
131  AliasAnalysis *AA;
132 
133  // state
134  std::unique_ptr<Spiller> SpillerInstance;
135  PQueue Queue;
136  unsigned NextCascade;
137 
138  // Live ranges pass through a number of stages as we try to allocate them.
139  // Some of the stages may also create new live ranges:
140  //
141  // - Region splitting.
142  // - Per-block splitting.
143  // - Local splitting.
144  // - Spilling.
145  //
146  // Ranges produced by one of the stages skip the previous stages when they are
147  // dequeued. This improves performance because we can skip interference checks
148  // that are unlikely to give any results. It also guarantees that the live
149  // range splitting algorithm terminates, something that is otherwise hard to
150  // ensure.
151  enum LiveRangeStage {
152  /// Newly created live range that has never been queued.
153  RS_New,
154 
155  /// Only attempt assignment and eviction. Then requeue as RS_Split.
156  RS_Assign,
157 
158  /// Attempt live range splitting if assignment is impossible.
159  RS_Split,
160 
161  /// Attempt more aggressive live range splitting that is guaranteed to make
162  /// progress. This is used for split products that may not be making
163  /// progress.
164  RS_Split2,
165 
166  /// Live range will be spilled. No more splitting will be attempted.
167  RS_Spill,
168 
169 
170  /// Live range is in memory. Because of other evictions, it might get moved
171  /// in a register in the end.
172  RS_Memory,
173 
174  /// There is nothing more we can do to this live range. Abort compilation
175  /// if it can't be assigned.
176  RS_Done
177  };
178 
179  // Enum CutOffStage to keep a track whether the register allocation failed
180  // because of the cutoffs encountered in last chance recoloring.
181  // Note: This is used as bitmask. New value should be next power of 2.
182  enum CutOffStage {
183  // No cutoffs encountered
184  CO_None = 0,
185 
186  // lcr-max-depth cutoff encountered
187  CO_Depth = 1,
188 
189  // lcr-max-interf cutoff encountered
190  CO_Interf = 2
191  };
192 
193  uint8_t CutOffInfo;
194 
195 #ifndef NDEBUG
196  static const char *const StageName[];
197 #endif
198 
199  // RegInfo - Keep additional information about each live range.
200  struct RegInfo {
201  LiveRangeStage Stage;
202 
203  // Cascade - Eviction loop prevention. See canEvictInterference().
204  unsigned Cascade;
205 
206  RegInfo() : Stage(RS_New), Cascade(0) {}
207  };
208 
210 
211  LiveRangeStage getStage(const LiveInterval &VirtReg) const {
212  return ExtraRegInfo[VirtReg.reg].Stage;
213  }
214 
215  void setStage(const LiveInterval &VirtReg, LiveRangeStage Stage) {
216  ExtraRegInfo.resize(MRI->getNumVirtRegs());
217  ExtraRegInfo[VirtReg.reg].Stage = Stage;
218  }
219 
220  template<typename Iterator>
221  void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) {
222  ExtraRegInfo.resize(MRI->getNumVirtRegs());
223  for (;Begin != End; ++Begin) {
224  unsigned Reg = *Begin;
225  if (ExtraRegInfo[Reg].Stage == RS_New)
226  ExtraRegInfo[Reg].Stage = NewStage;
227  }
228  }
229 
230  /// Cost of evicting interference.
231  struct EvictionCost {
232  unsigned BrokenHints; ///< Total number of broken hints.
233  float MaxWeight; ///< Maximum spill weight evicted.
234 
235  EvictionCost(): BrokenHints(0), MaxWeight(0) {}
236 
237  bool isMax() const { return BrokenHints == ~0u; }
238 
239  void setMax() { BrokenHints = ~0u; }
240 
241  void setBrokenHints(unsigned NHints) { BrokenHints = NHints; }
242 
243  bool operator<(const EvictionCost &O) const {
244  return std::tie(BrokenHints, MaxWeight) <
245  std::tie(O.BrokenHints, O.MaxWeight);
246  }
247  };
248 
249  // splitting state.
250  std::unique_ptr<SplitAnalysis> SA;
251  std::unique_ptr<SplitEditor> SE;
252 
253  /// Cached per-block interference maps
254  InterferenceCache IntfCache;
255 
256  /// All basic blocks where the current register has uses.
258 
259  /// Global live range splitting candidate info.
260  struct GlobalSplitCandidate {
261  // Register intended for assignment, or 0.
262  unsigned PhysReg;
263 
264  // SplitKit interval index for this candidate.
265  unsigned IntvIdx;
266 
267  // Interference for PhysReg.
269 
270  // Bundles where this candidate should be live.
271  BitVector LiveBundles;
272  SmallVector<unsigned, 8> ActiveBlocks;
273 
274  void reset(InterferenceCache &Cache, unsigned Reg) {
275  PhysReg = Reg;
276  IntvIdx = 0;
277  Intf.setPhysReg(Cache, Reg);
278  LiveBundles.clear();
279  ActiveBlocks.clear();
280  }
281 
282  // Set B[i] = C for every live bundle where B[i] was NoCand.
283  unsigned getBundles(SmallVectorImpl<unsigned> &B, unsigned C) {
284  unsigned Count = 0;
285  for (int i = LiveBundles.find_first(); i >= 0;
286  i = LiveBundles.find_next(i))
287  if (B[i] == NoCand) {
288  B[i] = C;
289  Count++;
290  }
291  return Count;
292  }
293  };
294 
295  /// Candidate info for each PhysReg in AllocationOrder.
296  /// This vector never shrinks, but grows to the size of the largest register
297  /// class.
299 
300  enum : unsigned { NoCand = ~0u };
301 
302  /// Candidate map. Each edge bundle is assigned to a GlobalCand entry, or to
303  /// NoCand which indicates the stack interval.
304  SmallVector<unsigned, 32> BundleCand;
305 
306  /// Callee-save register cost, calculated once per machine function.
307  BlockFrequency CSRCost;
308 
309  /// Run or not the local reassignment heuristic. This information is
310  /// obtained from the TargetSubtargetInfo.
311  bool EnableLocalReassign;
312 
313  /// Set of broken hints that may be reconciled later because of eviction.
314  SmallSetVector<LiveInterval *, 8> SetOfBrokenHints;
315 
316 public:
317  RAGreedy();
318 
319  /// Return the pass name.
320  StringRef getPassName() const override { return "Greedy Register Allocator"; }
321 
322  /// RAGreedy analysis usage.
323  void getAnalysisUsage(AnalysisUsage &AU) const override;
324  void releaseMemory() override;
325  Spiller &spiller() override { return *SpillerInstance; }
326  void enqueue(LiveInterval *LI) override;
327  LiveInterval *dequeue() override;
328  unsigned selectOrSplit(LiveInterval&, SmallVectorImpl<unsigned>&) override;
329  void aboutToRemoveInterval(LiveInterval &) override;
330 
331  /// Perform register allocation.
332  bool runOnMachineFunction(MachineFunction &mf) override;
333 
334  MachineFunctionProperties getRequiredProperties() const override {
337  }
338 
339  static char ID;
340 
341 private:
342  unsigned selectOrSplitImpl(LiveInterval &, SmallVectorImpl<unsigned> &,
343  SmallVirtRegSet &, unsigned = 0);
344 
345  bool LRE_CanEraseVirtReg(unsigned) override;
346  void LRE_WillShrinkVirtReg(unsigned) override;
347  void LRE_DidCloneVirtReg(unsigned, unsigned) override;
348  void enqueue(PQueue &CurQueue, LiveInterval *LI);
349  LiveInterval *dequeue(PQueue &CurQueue);
350 
351  BlockFrequency calcSpillCost();
352  bool addSplitConstraints(InterferenceCache::Cursor, BlockFrequency&);
353  void addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>);
354  void growRegion(GlobalSplitCandidate &Cand);
355  BlockFrequency calcGlobalSplitCost(GlobalSplitCandidate&);
356  bool calcCompactRegion(GlobalSplitCandidate&);
357  void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>);
358  void calcGapWeights(unsigned, SmallVectorImpl<float>&);
359  unsigned canReassign(LiveInterval &VirtReg, unsigned PhysReg);
360  bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool);
361  bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&);
362  void evictInterference(LiveInterval&, unsigned,
364  bool mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg,
365  SmallLISet &RecoloringCandidates,
366  const SmallVirtRegSet &FixedRegisters);
367 
368  unsigned tryAssign(LiveInterval&, AllocationOrder&,
370  unsigned tryEvict(LiveInterval&, AllocationOrder&,
371  SmallVectorImpl<unsigned>&, unsigned = ~0u);
372  unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
374  /// Calculate cost of region splitting.
375  unsigned calculateRegionSplitCost(LiveInterval &VirtReg,
376  AllocationOrder &Order,
377  BlockFrequency &BestCost,
378  unsigned &NumCands, bool IgnoreCSR);
379  /// Perform region splitting.
380  unsigned doRegionSplit(LiveInterval &VirtReg, unsigned BestCand,
381  bool HasCompact,
382  SmallVectorImpl<unsigned> &NewVRegs);
383  /// Check other options before using a callee-saved register for the first
384  /// time.
385  unsigned tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order,
386  unsigned PhysReg, unsigned &CostPerUseLimit,
387  SmallVectorImpl<unsigned> &NewVRegs);
388  void initializeCSRCost();
389  unsigned tryBlockSplit(LiveInterval&, AllocationOrder&,
391  unsigned tryInstructionSplit(LiveInterval&, AllocationOrder&,
393  unsigned tryLocalSplit(LiveInterval&, AllocationOrder&,
395  unsigned trySplit(LiveInterval&, AllocationOrder&,
397  unsigned tryLastChanceRecoloring(LiveInterval &, AllocationOrder &,
399  SmallVirtRegSet &, unsigned);
400  bool tryRecoloringCandidates(PQueue &, SmallVectorImpl<unsigned> &,
401  SmallVirtRegSet &, unsigned);
402  void tryHintRecoloring(LiveInterval &);
403  void tryHintsRecoloring();
404 
405  /// Model the information carried by one end of a copy.
406  struct HintInfo {
407  /// The frequency of the copy.
408  BlockFrequency Freq;
409  /// The virtual register or physical register.
410  unsigned Reg;
411  /// Its currently assigned register.
412  /// In case of a physical register Reg == PhysReg.
413  unsigned PhysReg;
414  HintInfo(BlockFrequency Freq, unsigned Reg, unsigned PhysReg)
415  : Freq(Freq), Reg(Reg), PhysReg(PhysReg) {}
416  };
417  typedef SmallVector<HintInfo, 4> HintsInfo;
418  BlockFrequency getBrokenHintFreq(const HintsInfo &, unsigned);
419  void collectHintInfo(unsigned, HintsInfo &);
420 
421  bool isUnusedCalleeSavedReg(unsigned PhysReg) const;
422 };
423 } // end anonymous namespace
424 
425 char RAGreedy::ID = 0;
427 
428 INITIALIZE_PASS_BEGIN(RAGreedy, "greedy",
429  "Greedy Register Allocator", false, false)
433 INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
434 INITIALIZE_PASS_DEPENDENCY(MachineScheduler)
443  "Greedy Register Allocator", false, false)
444 
445 #ifndef NDEBUG
446 const char *const RAGreedy::StageName[] = {
447  "RS_New",
448  "RS_Assign",
449  "RS_Split",
450  "RS_Split2",
451  "RS_Spill",
452  "RS_Memory",
453  "RS_Done"
454 };
455 #endif
456 
457 // Hysteresis to use when comparing floats.
458 // This helps stabilize decisions based on float comparisons.
459 const float Hysteresis = (2007 / 2048.0f); // 0.97998046875
460 
461 
463  return new RAGreedy();
464 }
465 
466 RAGreedy::RAGreedy(): MachineFunctionPass(ID) {
467 }
468 
469 void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
470  AU.setPreservesCFG();
477  AU.addRequired<SlotIndexes>();
481  AU.addRequired<LiveStacks>();
482  AU.addPreserved<LiveStacks>();
487  AU.addRequired<VirtRegMap>();
488  AU.addPreserved<VirtRegMap>();
491  AU.addRequired<EdgeBundles>();
493  MachineFunctionPass::getAnalysisUsage(AU);
494 }
495 
496 
497 //===----------------------------------------------------------------------===//
498 // LiveRangeEdit delegate methods
499 //===----------------------------------------------------------------------===//
500 
501 bool RAGreedy::LRE_CanEraseVirtReg(unsigned VirtReg) {
502  if (VRM->hasPhys(VirtReg)) {
503  LiveInterval &LI = LIS->getInterval(VirtReg);
504  Matrix->unassign(LI);
505  aboutToRemoveInterval(LI);
506  return true;
507  }
508  // Unassigned virtreg is probably in the priority queue.
509  // RegAllocBase will erase it after dequeueing.
510  return false;
511 }
512 
513 void RAGreedy::LRE_WillShrinkVirtReg(unsigned VirtReg) {
514  if (!VRM->hasPhys(VirtReg))
515  return;
516 
517  // Register is assigned, put it back on the queue for reassignment.
518  LiveInterval &LI = LIS->getInterval(VirtReg);
519  Matrix->unassign(LI);
520  enqueue(&LI);
521 }
522 
523 void RAGreedy::LRE_DidCloneVirtReg(unsigned New, unsigned Old) {
524  // Cloning a register we haven't even heard about yet? Just ignore it.
525  if (!ExtraRegInfo.inBounds(Old))
526  return;
527 
528  // LRE may clone a virtual register because dead code elimination causes it to
529  // be split into connected components. The new components are much smaller
530  // than the original, so they should get a new chance at being assigned.
531  // same stage as the parent.
532  ExtraRegInfo[Old].Stage = RS_Assign;
533  ExtraRegInfo.grow(New);
534  ExtraRegInfo[New] = ExtraRegInfo[Old];
535 }
536 
537 void RAGreedy::releaseMemory() {
538  SpillerInstance.reset();
539  ExtraRegInfo.clear();
540  GlobalCand.clear();
541 }
542 
543 void RAGreedy::enqueue(LiveInterval *LI) { enqueue(Queue, LI); }
544 
545 void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) {
546  // Prioritize live ranges by size, assigning larger ranges first.
547  // The queue holds (size, reg) pairs.
548  const unsigned Size = LI->getSize();
549  const unsigned Reg = LI->reg;
550  assert(TargetRegisterInfo::isVirtualRegister(Reg) &&
551  "Can only enqueue virtual registers");
552  unsigned Prio;
553 
554  ExtraRegInfo.grow(Reg);
555  if (ExtraRegInfo[Reg].Stage == RS_New)
556  ExtraRegInfo[Reg].Stage = RS_Assign;
557 
558  if (ExtraRegInfo[Reg].Stage == RS_Split) {
559  // Unsplit ranges that couldn't be allocated immediately are deferred until
560  // everything else has been allocated.
561  Prio = Size;
562  } else if (ExtraRegInfo[Reg].Stage == RS_Memory) {
563  // Memory operand should be considered last.
564  // Change the priority such that Memory operand are assigned in
565  // the reverse order that they came in.
566  // TODO: Make this a member variable and probably do something about hints.
567  static unsigned MemOp = 0;
568  Prio = MemOp++;
569  } else {
570  // Giant live ranges fall back to the global assignment heuristic, which
571  // prevents excessive spilling in pathological cases.
572  bool ReverseLocal = TRI->reverseLocalAssignment();
573  const TargetRegisterClass &RC = *MRI->getRegClass(Reg);
574  bool ForceGlobal = !ReverseLocal &&
575  (Size / SlotIndex::InstrDist) > (2 * RC.getNumRegs());
576 
577  if (ExtraRegInfo[Reg].Stage == RS_Assign && !ForceGlobal && !LI->empty() &&
578  LIS->intervalIsInOneMBB(*LI)) {
579  // Allocate original local ranges in linear instruction order. Since they
580  // are singly defined, this produces optimal coloring in the absence of
581  // global interference and other constraints.
582  if (!ReverseLocal)
583  Prio = LI->beginIndex().getInstrDistance(Indexes->getLastIndex());
584  else {
585  // Allocating bottom up may allow many short LRGs to be assigned first
586  // to one of the cheap registers. This could be much faster for very
587  // large blocks on targets with many physical registers.
588  Prio = Indexes->getZeroIndex().getInstrDistance(LI->endIndex());
589  }
590  Prio |= RC.AllocationPriority << 24;
591  } else {
592  // Allocate global and split ranges in long->short order. Long ranges that
593  // don't fit should be spilled (or split) ASAP so they don't create
594  // interference. Mark a bit to prioritize global above local ranges.
595  Prio = (1u << 29) + Size;
596  }
597  // Mark a higher bit to prioritize global and local above RS_Split.
598  Prio |= (1u << 31);
599 
600  // Boost ranges that have a physical register hint.
601  if (VRM->hasKnownPreference(Reg))
602  Prio |= (1u << 30);
603  }
604  // The virtual register number is a tie breaker for same-sized ranges.
605  // Give lower vreg numbers higher priority to assign them first.
606  CurQueue.push(std::make_pair(Prio, ~Reg));
607 }
608 
609 LiveInterval *RAGreedy::dequeue() { return dequeue(Queue); }
610 
611 LiveInterval *RAGreedy::dequeue(PQueue &CurQueue) {
612  if (CurQueue.empty())
613  return nullptr;
614  LiveInterval *LI = &LIS->getInterval(~CurQueue.top().second);
615  CurQueue.pop();
616  return LI;
617 }
618 
619 
620 //===----------------------------------------------------------------------===//
621 // Direct Assignment
622 //===----------------------------------------------------------------------===//
623 
624 /// tryAssign - Try to assign VirtReg to an available register.
625 unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
626  AllocationOrder &Order,
627  SmallVectorImpl<unsigned> &NewVRegs) {
628  Order.rewind();
629  unsigned PhysReg;
630  while ((PhysReg = Order.next()))
631  if (!Matrix->checkInterference(VirtReg, PhysReg))
632  break;
633  if (!PhysReg || Order.isHint())
634  return PhysReg;
635 
636  // PhysReg is available, but there may be a better choice.
637 
638  // If we missed a simple hint, try to cheaply evict interference from the
639  // preferred register.
640  if (unsigned Hint = MRI->getSimpleHint(VirtReg.reg))
641  if (Order.isHint(Hint)) {
642  DEBUG(dbgs() << "missed hint " << PrintReg(Hint, TRI) << '\n');
643  EvictionCost MaxCost;
644  MaxCost.setBrokenHints(1);
645  if (canEvictInterference(VirtReg, Hint, true, MaxCost)) {
646  evictInterference(VirtReg, Hint, NewVRegs);
647  return Hint;
648  }
649  // Record the missed hint, we may be able to recover
650  // at the end if the surrounding allocation changed.
651  SetOfBrokenHints.insert(&VirtReg);
652  }
653 
654  // Try to evict interference from a cheaper alternative.
655  unsigned Cost = TRI->getCostPerUse(PhysReg);
656 
657  // Most registers have 0 additional cost.
658  if (!Cost)
659  return PhysReg;
660 
661  DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is available at cost " << Cost
662  << '\n');
663  unsigned CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost);
664  return CheapReg ? CheapReg : PhysReg;
665 }
666 
667 
668 //===----------------------------------------------------------------------===//
669 // Interference eviction
670 //===----------------------------------------------------------------------===//
671 
672 unsigned RAGreedy::canReassign(LiveInterval &VirtReg, unsigned PrevReg) {
673  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
674  unsigned PhysReg;
675  while ((PhysReg = Order.next())) {
676  if (PhysReg == PrevReg)
677  continue;
678 
679  MCRegUnitIterator Units(PhysReg, TRI);
680  for (; Units.isValid(); ++Units) {
681  // Instantiate a "subquery", not to be confused with the Queries array.
682  LiveIntervalUnion::Query subQ(&VirtReg, &Matrix->getLiveUnions()[*Units]);
683  if (subQ.checkInterference())
684  break;
685  }
686  // If no units have interference, break out with the current PhysReg.
687  if (!Units.isValid())
688  break;
689  }
690  if (PhysReg)
691  DEBUG(dbgs() << "can reassign: " << VirtReg << " from "
692  << PrintReg(PrevReg, TRI) << " to " << PrintReg(PhysReg, TRI)
693  << '\n');
694  return PhysReg;
695 }
696 
697 /// shouldEvict - determine if A should evict the assigned live range B. The
698 /// eviction policy defined by this function together with the allocation order
699 /// defined by enqueue() decides which registers ultimately end up being split
700 /// and spilled.
701 ///
702 /// Cascade numbers are used to prevent infinite loops if this function is a
703 /// cyclic relation.
704 ///
705 /// @param A The live range to be assigned.
706 /// @param IsHint True when A is about to be assigned to its preferred
707 /// register.
708 /// @param B The live range to be evicted.
709 /// @param BreaksHint True when B is already assigned to its preferred register.
710 bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
711  LiveInterval &B, bool BreaksHint) {
712  bool CanSplit = getStage(B) < RS_Spill;
713 
714  // Be fairly aggressive about following hints as long as the evictee can be
715  // split.
716  if (CanSplit && IsHint && !BreaksHint)
717  return true;
718 
719  if (A.weight > B.weight) {
720  DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight << '\n');
721  return true;
722  }
723  return false;
724 }
725 
726 /// canEvictInterference - Return true if all interferences between VirtReg and
727 /// PhysReg can be evicted.
728 ///
729 /// @param VirtReg Live range that is about to be assigned.
730 /// @param PhysReg Desired register for assignment.
731 /// @param IsHint True when PhysReg is VirtReg's preferred register.
732 /// @param MaxCost Only look for cheaper candidates and update with new cost
733 /// when returning true.
734 /// @returns True when interference can be evicted cheaper than MaxCost.
735 bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg,
736  bool IsHint, EvictionCost &MaxCost) {
737  // It is only possible to evict virtual register interference.
738  if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg)
739  return false;
740 
741  bool IsLocal = LIS->intervalIsInOneMBB(VirtReg);
742 
743  // Find VirtReg's cascade number. This will be unassigned if VirtReg was never
744  // involved in an eviction before. If a cascade number was assigned, deny
745  // evicting anything with the same or a newer cascade number. This prevents
746  // infinite eviction loops.
747  //
748  // This works out so a register without a cascade number is allowed to evict
749  // anything, and it can be evicted by anything.
750  unsigned Cascade = ExtraRegInfo[VirtReg.reg].Cascade;
751  if (!Cascade)
752  Cascade = NextCascade;
753 
754  EvictionCost Cost;
755  for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
756  LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units);
757  // If there is 10 or more interferences, chances are one is heavier.
758  if (Q.collectInterferingVRegs(10) >= 10)
759  return false;
760 
761  // Check if any interfering live range is heavier than MaxWeight.
762  for (unsigned i = Q.interferingVRegs().size(); i; --i) {
763  LiveInterval *Intf = Q.interferingVRegs()[i - 1];
764  assert(TargetRegisterInfo::isVirtualRegister(Intf->reg) &&
765  "Only expecting virtual register interference from query");
766  // Never evict spill products. They cannot split or spill.
767  if (getStage(*Intf) == RS_Done)
768  return false;
769  // Once a live range becomes small enough, it is urgent that we find a
770  // register for it. This is indicated by an infinite spill weight. These
771  // urgent live ranges get to evict almost anything.
772  //
773  // Also allow urgent evictions of unspillable ranges from a strictly
774  // larger allocation order.
775  bool Urgent = !VirtReg.isSpillable() &&
776  (Intf->isSpillable() ||
777  RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg)) <
778  RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(Intf->reg)));
779  // Only evict older cascades or live ranges without a cascade.
780  unsigned IntfCascade = ExtraRegInfo[Intf->reg].Cascade;
781  if (Cascade <= IntfCascade) {
782  if (!Urgent)
783  return false;
784  // We permit breaking cascades for urgent evictions. It should be the
785  // last resort, though, so make it really expensive.
786  Cost.BrokenHints += 10;
787  }
788  // Would this break a satisfied hint?
789  bool BreaksHint = VRM->hasPreferredPhys(Intf->reg);
790  // Update eviction cost.
791  Cost.BrokenHints += BreaksHint;
792  Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight);
793  // Abort if this would be too expensive.
794  if (!(Cost < MaxCost))
795  return false;
796  if (Urgent)
797  continue;
798  // Apply the eviction policy for non-urgent evictions.
799  if (!shouldEvict(VirtReg, IsHint, *Intf, BreaksHint))
800  return false;
801  // If !MaxCost.isMax(), then we're just looking for a cheap register.
802  // Evicting another local live range in this case could lead to suboptimal
803  // coloring.
804  if (!MaxCost.isMax() && IsLocal && LIS->intervalIsInOneMBB(*Intf) &&
805  (!EnableLocalReassign || !canReassign(*Intf, PhysReg))) {
806  return false;
807  }
808  }
809  }
810  MaxCost = Cost;
811  return true;
812 }
813 
814 /// evictInterference - Evict any interferring registers that prevent VirtReg
815 /// from being assigned to Physreg. This assumes that canEvictInterference
816 /// returned true.
817 void RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg,
818  SmallVectorImpl<unsigned> &NewVRegs) {
819  // Make sure that VirtReg has a cascade number, and assign that cascade
820  // number to every evicted register. These live ranges than then only be
821  // evicted by a newer cascade, preventing infinite loops.
822  unsigned Cascade = ExtraRegInfo[VirtReg.reg].Cascade;
823  if (!Cascade)
824  Cascade = ExtraRegInfo[VirtReg.reg].Cascade = NextCascade++;
825 
826  DEBUG(dbgs() << "evicting " << PrintReg(PhysReg, TRI)
827  << " interference: Cascade " << Cascade << '\n');
828 
829  // Collect all interfering virtregs first.
831  for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
832  LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units);
833  assert(Q.seenAllInterferences() && "Didn't check all interfererences.");
835  Intfs.append(IVR.begin(), IVR.end());
836  }
837 
838  // Evict them second. This will invalidate the queries.
839  for (unsigned i = 0, e = Intfs.size(); i != e; ++i) {
840  LiveInterval *Intf = Intfs[i];
841  // The same VirtReg may be present in multiple RegUnits. Skip duplicates.
842  if (!VRM->hasPhys(Intf->reg))
843  continue;
844  Matrix->unassign(*Intf);
845  assert((ExtraRegInfo[Intf->reg].Cascade < Cascade ||
846  VirtReg.isSpillable() < Intf->isSpillable()) &&
847  "Cannot decrease cascade number, illegal eviction");
848  ExtraRegInfo[Intf->reg].Cascade = Cascade;
849  ++NumEvicted;
850  NewVRegs.push_back(Intf->reg);
851  }
852 }
853 
854 /// Returns true if the given \p PhysReg is a callee saved register and has not
855 /// been used for allocation yet.
856 bool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const {
857  unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg);
858  if (CSR == 0)
859  return false;
860 
861  return !Matrix->isPhysRegUsed(PhysReg);
862 }
863 
864 /// tryEvict - Try to evict all interferences for a physreg.
865 /// @param VirtReg Currently unassigned virtual register.
866 /// @param Order Physregs to try.
867 /// @return Physreg to assign VirtReg, or 0.
868 unsigned RAGreedy::tryEvict(LiveInterval &VirtReg,
869  AllocationOrder &Order,
870  SmallVectorImpl<unsigned> &NewVRegs,
871  unsigned CostPerUseLimit) {
872  NamedRegionTimer T("evict", "Evict", TimerGroupName, TimerGroupDescription,
874 
875  // Keep track of the cheapest interference seen so far.
876  EvictionCost BestCost;
877  BestCost.setMax();
878  unsigned BestPhys = 0;
879  unsigned OrderLimit = Order.getOrder().size();
880 
881  // When we are just looking for a reduced cost per use, don't break any
882  // hints, and only evict smaller spill weights.
883  if (CostPerUseLimit < ~0u) {
884  BestCost.BrokenHints = 0;
885  BestCost.MaxWeight = VirtReg.weight;
886 
887  // Check of any registers in RC are below CostPerUseLimit.
888  const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg);
889  unsigned MinCost = RegClassInfo.getMinCost(RC);
890  if (MinCost >= CostPerUseLimit) {
891  DEBUG(dbgs() << TRI->getRegClassName(RC) << " minimum cost = " << MinCost
892  << ", no cheaper registers to be found.\n");
893  return 0;
894  }
895 
896  // It is normal for register classes to have a long tail of registers with
897  // the same cost. We don't need to look at them if they're too expensive.
898  if (TRI->getCostPerUse(Order.getOrder().back()) >= CostPerUseLimit) {
899  OrderLimit = RegClassInfo.getLastCostChange(RC);
900  DEBUG(dbgs() << "Only trying the first " << OrderLimit << " regs.\n");
901  }
902  }
903 
904  Order.rewind();
905  while (unsigned PhysReg = Order.next(OrderLimit)) {
906  if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit)
907  continue;
908  // The first use of a callee-saved register in a function has cost 1.
909  // Don't start using a CSR when the CostPerUseLimit is low.
910  if (CostPerUseLimit == 1 && isUnusedCalleeSavedReg(PhysReg)) {
911  DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " would clobber CSR "
912  << PrintReg(RegClassInfo.getLastCalleeSavedAlias(PhysReg), TRI)
913  << '\n');
914  continue;
915  }
916 
917  if (!canEvictInterference(VirtReg, PhysReg, false, BestCost))
918  continue;
919 
920  // Best so far.
921  BestPhys = PhysReg;
922 
923  // Stop if the hint can be used.
924  if (Order.isHint())
925  break;
926  }
927 
928  if (!BestPhys)
929  return 0;
930 
931  evictInterference(VirtReg, BestPhys, NewVRegs);
932  return BestPhys;
933 }
934 
935 
936 //===----------------------------------------------------------------------===//
937 // Region Splitting
938 //===----------------------------------------------------------------------===//
939 
940 /// addSplitConstraints - Fill out the SplitConstraints vector based on the
941 /// interference pattern in Physreg and its aliases. Add the constraints to
942 /// SpillPlacement and return the static cost of this split in Cost, assuming
943 /// that all preferences in SplitConstraints are met.
944 /// Return false if there are no bundles with positive bias.
945 bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,
946  BlockFrequency &Cost) {
947  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
948 
949  // Reset interference dependent info.
950  SplitConstraints.resize(UseBlocks.size());
951  BlockFrequency StaticCost = 0;
952  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
953  const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
954  SpillPlacement::BlockConstraint &BC = SplitConstraints[i];
955 
956  BC.Number = BI.MBB->getNumber();
957  Intf.moveToBlock(BC.Number);
958  BC.Entry = BI.LiveIn ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
959  BC.Exit = BI.LiveOut ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
960  BC.ChangesValue = BI.FirstDef.isValid();
961 
962  if (!Intf.hasInterference())
963  continue;
964 
965  // Number of spill code instructions to insert.
966  unsigned Ins = 0;
967 
968  // Interference for the live-in value.
969  if (BI.LiveIn) {
970  if (Intf.first() <= Indexes->getMBBStartIdx(BC.Number)) {
971  BC.Entry = SpillPlacement::MustSpill;
972  ++Ins;
973  } else if (Intf.first() < BI.FirstInstr) {
974  BC.Entry = SpillPlacement::PrefSpill;
975  ++Ins;
976  } else if (Intf.first() < BI.LastInstr) {
977  ++Ins;
978  }
979  }
980 
981  // Interference for the live-out value.
982  if (BI.LiveOut) {
983  if (Intf.last() >= SA->getLastSplitPoint(BC.Number)) {
984  BC.Exit = SpillPlacement::MustSpill;
985  ++Ins;
986  } else if (Intf.last() > BI.LastInstr) {
987  BC.Exit = SpillPlacement::PrefSpill;
988  ++Ins;
989  } else if (Intf.last() > BI.FirstInstr) {
990  ++Ins;
991  }
992  }
993 
994  // Accumulate the total frequency of inserted spill code.
995  while (Ins--)
996  StaticCost += SpillPlacer->getBlockFrequency(BC.Number);
997  }
998  Cost = StaticCost;
999 
1000  // Add constraints for use-blocks. Note that these are the only constraints
1001  // that may add a positive bias, it is downhill from here.
1002  SpillPlacer->addConstraints(SplitConstraints);
1003  return SpillPlacer->scanActiveBundles();
1004 }
1005 
1006 
1007 /// addThroughConstraints - Add constraints and links to SpillPlacer from the
1008 /// live-through blocks in Blocks.
1009 void RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf,
1010  ArrayRef<unsigned> Blocks) {
1011  const unsigned GroupSize = 8;
1012  SpillPlacement::BlockConstraint BCS[GroupSize];
1013  unsigned TBS[GroupSize];
1014  unsigned B = 0, T = 0;
1015 
1016  for (unsigned i = 0; i != Blocks.size(); ++i) {
1017  unsigned Number = Blocks[i];
1018  Intf.moveToBlock(Number);
1019 
1020  if (!Intf.hasInterference()) {
1021  assert(T < GroupSize && "Array overflow");
1022  TBS[T] = Number;
1023  if (++T == GroupSize) {
1024  SpillPlacer->addLinks(makeArrayRef(TBS, T));
1025  T = 0;
1026  }
1027  continue;
1028  }
1029 
1030  assert(B < GroupSize && "Array overflow");
1031  BCS[B].Number = Number;
1032 
1033  // Interference for the live-in value.
1034  if (Intf.first() <= Indexes->getMBBStartIdx(Number))
1035  BCS[B].Entry = SpillPlacement::MustSpill;
1036  else
1037  BCS[B].Entry = SpillPlacement::PrefSpill;
1038 
1039  // Interference for the live-out value.
1040  if (Intf.last() >= SA->getLastSplitPoint(Number))
1041  BCS[B].Exit = SpillPlacement::MustSpill;
1042  else
1043  BCS[B].Exit = SpillPlacement::PrefSpill;
1044 
1045  if (++B == GroupSize) {
1046  SpillPlacer->addConstraints(makeArrayRef(BCS, B));
1047  B = 0;
1048  }
1049  }
1050 
1051  SpillPlacer->addConstraints(makeArrayRef(BCS, B));
1052  SpillPlacer->addLinks(makeArrayRef(TBS, T));
1053 }
1054 
1055 void RAGreedy::growRegion(GlobalSplitCandidate &Cand) {
1056  // Keep track of through blocks that have not been added to SpillPlacer.
1057  BitVector Todo = SA->getThroughBlocks();
1058  SmallVectorImpl<unsigned> &ActiveBlocks = Cand.ActiveBlocks;
1059  unsigned AddedTo = 0;
1060 #ifndef NDEBUG
1061  unsigned Visited = 0;
1062 #endif
1063 
1064  for (;;) {
1065  ArrayRef<unsigned> NewBundles = SpillPlacer->getRecentPositive();
1066  // Find new through blocks in the periphery of PrefRegBundles.
1067  for (int i = 0, e = NewBundles.size(); i != e; ++i) {
1068  unsigned Bundle = NewBundles[i];
1069  // Look at all blocks connected to Bundle in the full graph.
1070  ArrayRef<unsigned> Blocks = Bundles->getBlocks(Bundle);
1071  for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end();
1072  I != E; ++I) {
1073  unsigned Block = *I;
1074  if (!Todo.test(Block))
1075  continue;
1076  Todo.reset(Block);
1077  // This is a new through block. Add it to SpillPlacer later.
1078  ActiveBlocks.push_back(Block);
1079 #ifndef NDEBUG
1080  ++Visited;
1081 #endif
1082  }
1083  }
1084  // Any new blocks to add?
1085  if (ActiveBlocks.size() == AddedTo)
1086  break;
1087 
1088  // Compute through constraints from the interference, or assume that all
1089  // through blocks prefer spilling when forming compact regions.
1090  auto NewBlocks = makeArrayRef(ActiveBlocks).slice(AddedTo);
1091  if (Cand.PhysReg)
1092  addThroughConstraints(Cand.Intf, NewBlocks);
1093  else
1094  // Provide a strong negative bias on through blocks to prevent unwanted
1095  // liveness on loop backedges.
1096  SpillPlacer->addPrefSpill(NewBlocks, /* Strong= */ true);
1097  AddedTo = ActiveBlocks.size();
1098 
1099  // Perhaps iterating can enable more bundles?
1100  SpillPlacer->iterate();
1101  }
1102  DEBUG(dbgs() << ", v=" << Visited);
1103 }
1104 
1105 /// calcCompactRegion - Compute the set of edge bundles that should be live
1106 /// when splitting the current live range into compact regions. Compact
1107 /// regions can be computed without looking at interference. They are the
1108 /// regions formed by removing all the live-through blocks from the live range.
1109 ///
1110 /// Returns false if the current live range is already compact, or if the
1111 /// compact regions would form single block regions anyway.
1112 bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) {
1113  // Without any through blocks, the live range is already compact.
1114  if (!SA->getNumThroughBlocks())
1115  return false;
1116 
1117  // Compact regions don't correspond to any physreg.
1118  Cand.reset(IntfCache, 0);
1119 
1120  DEBUG(dbgs() << "Compact region bundles");
1121 
1122  // Use the spill placer to determine the live bundles. GrowRegion pretends
1123  // that all the through blocks have interference when PhysReg is unset.
1124  SpillPlacer->prepare(Cand.LiveBundles);
1125 
1126  // The static split cost will be zero since Cand.Intf reports no interference.
1127  BlockFrequency Cost;
1128  if (!addSplitConstraints(Cand.Intf, Cost)) {
1129  DEBUG(dbgs() << ", none.\n");
1130  return false;
1131  }
1132 
1133  growRegion(Cand);
1134  SpillPlacer->finish();
1135 
1136  if (!Cand.LiveBundles.any()) {
1137  DEBUG(dbgs() << ", none.\n");
1138  return false;
1139  }
1140 
1141  DEBUG({
1142  for (int i = Cand.LiveBundles.find_first(); i>=0;
1143  i = Cand.LiveBundles.find_next(i))
1144  dbgs() << " EB#" << i;
1145  dbgs() << ".\n";
1146  });
1147  return true;
1148 }
1149 
1150 /// calcSpillCost - Compute how expensive it would be to split the live range in
1151 /// SA around all use blocks instead of forming bundle regions.
1152 BlockFrequency RAGreedy::calcSpillCost() {
1153  BlockFrequency Cost = 0;
1154  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
1155  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
1156  const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
1157  unsigned Number = BI.MBB->getNumber();
1158  // We normally only need one spill instruction - a load or a store.
1159  Cost += SpillPlacer->getBlockFrequency(Number);
1160 
1161  // Unless the value is redefined in the block.
1162  if (BI.LiveIn && BI.LiveOut && BI.FirstDef)
1163  Cost += SpillPlacer->getBlockFrequency(Number);
1164  }
1165  return Cost;
1166 }
1167 
1168 /// calcGlobalSplitCost - Return the global split cost of following the split
1169 /// pattern in LiveBundles. This cost should be added to the local cost of the
1170 /// interference pattern in SplitConstraints.
1171 ///
1172 BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) {
1173  BlockFrequency GlobalCost = 0;
1174  const BitVector &LiveBundles = Cand.LiveBundles;
1175  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
1176  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
1177  const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
1178  SpillPlacement::BlockConstraint &BC = SplitConstraints[i];
1179  bool RegIn = LiveBundles[Bundles->getBundle(BC.Number, 0)];
1180  bool RegOut = LiveBundles[Bundles->getBundle(BC.Number, 1)];
1181  unsigned Ins = 0;
1182 
1183  if (BI.LiveIn)
1184  Ins += RegIn != (BC.Entry == SpillPlacement::PrefReg);
1185  if (BI.LiveOut)
1186  Ins += RegOut != (BC.Exit == SpillPlacement::PrefReg);
1187  while (Ins--)
1188  GlobalCost += SpillPlacer->getBlockFrequency(BC.Number);
1189  }
1190 
1191  for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) {
1192  unsigned Number = Cand.ActiveBlocks[i];
1193  bool RegIn = LiveBundles[Bundles->getBundle(Number, 0)];
1194  bool RegOut = LiveBundles[Bundles->getBundle(Number, 1)];
1195  if (!RegIn && !RegOut)
1196  continue;
1197  if (RegIn && RegOut) {
1198  // We need double spill code if this block has interference.
1199  Cand.Intf.moveToBlock(Number);
1200  if (Cand.Intf.hasInterference()) {
1201  GlobalCost += SpillPlacer->getBlockFrequency(Number);
1202  GlobalCost += SpillPlacer->getBlockFrequency(Number);
1203  }
1204  continue;
1205  }
1206  // live-in / stack-out or stack-in live-out.
1207  GlobalCost += SpillPlacer->getBlockFrequency(Number);
1208  }
1209  return GlobalCost;
1210 }
1211 
1212 /// splitAroundRegion - Split the current live range around the regions
1213 /// determined by BundleCand and GlobalCand.
1214 ///
1215 /// Before calling this function, GlobalCand and BundleCand must be initialized
1216 /// so each bundle is assigned to a valid candidate, or NoCand for the
1217 /// stack-bound bundles. The shared SA/SE SplitAnalysis and SplitEditor
1218 /// objects must be initialized for the current live range, and intervals
1219 /// created for the used candidates.
1220 ///
1221 /// @param LREdit The LiveRangeEdit object handling the current split.
1222 /// @param UsedCands List of used GlobalCand entries. Every BundleCand value
1223 /// must appear in this list.
1224 void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
1225  ArrayRef<unsigned> UsedCands) {
1226  // These are the intervals created for new global ranges. We may create more
1227  // intervals for local ranges.
1228  const unsigned NumGlobalIntvs = LREdit.size();
1229  DEBUG(dbgs() << "splitAroundRegion with " << NumGlobalIntvs << " globals.\n");
1230  assert(NumGlobalIntvs && "No global intervals configured");
1231 
1232  // Isolate even single instructions when dealing with a proper sub-class.
1233  // That guarantees register class inflation for the stack interval because it
1234  // is all copies.
1235  unsigned Reg = SA->getParent().reg;
1236  bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
1237 
1238  // First handle all the blocks with uses.
1239  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
1240  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
1241  const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
1242  unsigned Number = BI.MBB->getNumber();
1243  unsigned IntvIn = 0, IntvOut = 0;
1244  SlotIndex IntfIn, IntfOut;
1245  if (BI.LiveIn) {
1246  unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)];
1247  if (CandIn != NoCand) {
1248  GlobalSplitCandidate &Cand = GlobalCand[CandIn];
1249  IntvIn = Cand.IntvIdx;
1250  Cand.Intf.moveToBlock(Number);
1251  IntfIn = Cand.Intf.first();
1252  }
1253  }
1254  if (BI.LiveOut) {
1255  unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)];
1256  if (CandOut != NoCand) {
1257  GlobalSplitCandidate &Cand = GlobalCand[CandOut];
1258  IntvOut = Cand.IntvIdx;
1259  Cand.Intf.moveToBlock(Number);
1260  IntfOut = Cand.Intf.last();
1261  }
1262  }
1263 
1264  // Create separate intervals for isolated blocks with multiple uses.
1265  if (!IntvIn && !IntvOut) {
1266  DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " isolated.\n");
1267  if (SA->shouldSplitSingleBlock(BI, SingleInstrs))
1268  SE->splitSingleBlock(BI);
1269  continue;
1270  }
1271 
1272  if (IntvIn && IntvOut)
1273  SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut);
1274  else if (IntvIn)
1275  SE->splitRegInBlock(BI, IntvIn, IntfIn);
1276  else
1277  SE->splitRegOutBlock(BI, IntvOut, IntfOut);
1278  }
1279 
1280  // Handle live-through blocks. The relevant live-through blocks are stored in
1281  // the ActiveBlocks list with each candidate. We need to filter out
1282  // duplicates.
1283  BitVector Todo = SA->getThroughBlocks();
1284  for (unsigned c = 0; c != UsedCands.size(); ++c) {
1285  ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks;
1286  for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
1287  unsigned Number = Blocks[i];
1288  if (!Todo.test(Number))
1289  continue;
1290  Todo.reset(Number);
1291 
1292  unsigned IntvIn = 0, IntvOut = 0;
1293  SlotIndex IntfIn, IntfOut;
1294 
1295  unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)];
1296  if (CandIn != NoCand) {
1297  GlobalSplitCandidate &Cand = GlobalCand[CandIn];
1298  IntvIn = Cand.IntvIdx;
1299  Cand.Intf.moveToBlock(Number);
1300  IntfIn = Cand.Intf.first();
1301  }
1302 
1303  unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)];
1304  if (CandOut != NoCand) {
1305  GlobalSplitCandidate &Cand = GlobalCand[CandOut];
1306  IntvOut = Cand.IntvIdx;
1307  Cand.Intf.moveToBlock(Number);
1308  IntfOut = Cand.Intf.last();
1309  }
1310  if (!IntvIn && !IntvOut)
1311  continue;
1312  SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut);
1313  }
1314  }
1315 
1316  ++NumGlobalSplits;
1317 
1318  SmallVector<unsigned, 8> IntvMap;
1319  SE->finish(&IntvMap);
1320  DebugVars->splitRegister(Reg, LREdit.regs(), *LIS);
1321 
1322  ExtraRegInfo.resize(MRI->getNumVirtRegs());
1323  unsigned OrigBlocks = SA->getNumLiveBlocks();
1324 
1325  // Sort out the new intervals created by splitting. We get four kinds:
1326  // - Remainder intervals should not be split again.
1327  // - Candidate intervals can be assigned to Cand.PhysReg.
1328  // - Block-local splits are candidates for local splitting.
1329  // - DCE leftovers should go back on the queue.
1330  for (unsigned i = 0, e = LREdit.size(); i != e; ++i) {
1331  LiveInterval &Reg = LIS->getInterval(LREdit.get(i));
1332 
1333  // Ignore old intervals from DCE.
1334  if (getStage(Reg) != RS_New)
1335  continue;
1336 
1337  // Remainder interval. Don't try splitting again, spill if it doesn't
1338  // allocate.
1339  if (IntvMap[i] == 0) {
1340  setStage(Reg, RS_Spill);
1341  continue;
1342  }
1343 
1344  // Global intervals. Allow repeated splitting as long as the number of live
1345  // blocks is strictly decreasing.
1346  if (IntvMap[i] < NumGlobalIntvs) {
1347  if (SA->countLiveBlocks(&Reg) >= OrigBlocks) {
1348  DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks
1349  << " blocks as original.\n");
1350  // Don't allow repeated splitting as a safe guard against looping.
1351  setStage(Reg, RS_Split2);
1352  }
1353  continue;
1354  }
1355 
1356  // Other intervals are treated as new. This includes local intervals created
1357  // for blocks with multiple uses, and anything created by DCE.
1358  }
1359 
1360  if (VerifyEnabled)
1361  MF->verify(this, "After splitting live range around region");
1362 }
1363 
1364 unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
1365  SmallVectorImpl<unsigned> &NewVRegs) {
1366  unsigned NumCands = 0;
1367  BlockFrequency BestCost;
1368 
1369  // Check if we can split this live range around a compact region.
1370  bool HasCompact = calcCompactRegion(GlobalCand.front());
1371  if (HasCompact) {
1372  // Yes, keep GlobalCand[0] as the compact region candidate.
1373  NumCands = 1;
1374  BestCost = BlockFrequency::getMaxFrequency();
1375  } else {
1376  // No benefit from the compact region, our fallback will be per-block
1377  // splitting. Make sure we find a solution that is cheaper than spilling.
1378  BestCost = calcSpillCost();
1379  DEBUG(dbgs() << "Cost of isolating all blocks = ";
1380  MBFI->printBlockFreq(dbgs(), BestCost) << '\n');
1381  }
1382 
1383  unsigned BestCand =
1384  calculateRegionSplitCost(VirtReg, Order, BestCost, NumCands,
1385  false/*IgnoreCSR*/);
1386 
1387  // No solutions found, fall back to single block splitting.
1388  if (!HasCompact && BestCand == NoCand)
1389  return 0;
1390 
1391  return doRegionSplit(VirtReg, BestCand, HasCompact, NewVRegs);
1392 }
1393 
1394 unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg,
1395  AllocationOrder &Order,
1396  BlockFrequency &BestCost,
1397  unsigned &NumCands,
1398  bool IgnoreCSR) {
1399  unsigned BestCand = NoCand;
1400  Order.rewind();
1401  while (unsigned PhysReg = Order.next()) {
1402  if (IgnoreCSR && isUnusedCalleeSavedReg(PhysReg))
1403  continue;
1404 
1405  // Discard bad candidates before we run out of interference cache cursors.
1406  // This will only affect register classes with a lot of registers (>32).
1407  if (NumCands == IntfCache.getMaxCursors()) {
1408  unsigned WorstCount = ~0u;
1409  unsigned Worst = 0;
1410  for (unsigned i = 0; i != NumCands; ++i) {
1411  if (i == BestCand || !GlobalCand[i].PhysReg)
1412  continue;
1413  unsigned Count = GlobalCand[i].LiveBundles.count();
1414  if (Count < WorstCount) {
1415  Worst = i;
1416  WorstCount = Count;
1417  }
1418  }
1419  --NumCands;
1420  GlobalCand[Worst] = GlobalCand[NumCands];
1421  if (BestCand == NumCands)
1422  BestCand = Worst;
1423  }
1424 
1425  if (GlobalCand.size() <= NumCands)
1426  GlobalCand.resize(NumCands+1);
1427  GlobalSplitCandidate &Cand = GlobalCand[NumCands];
1428  Cand.reset(IntfCache, PhysReg);
1429 
1430  SpillPlacer->prepare(Cand.LiveBundles);
1431  BlockFrequency Cost;
1432  if (!addSplitConstraints(Cand.Intf, Cost)) {
1433  DEBUG(dbgs() << PrintReg(PhysReg, TRI) << "\tno positive bundles\n");
1434  continue;
1435  }
1436  DEBUG(dbgs() << PrintReg(PhysReg, TRI) << "\tstatic = ";
1437  MBFI->printBlockFreq(dbgs(), Cost));
1438  if (Cost >= BestCost) {
1439  DEBUG({
1440  if (BestCand == NoCand)
1441  dbgs() << " worse than no bundles\n";
1442  else
1443  dbgs() << " worse than "
1444  << PrintReg(GlobalCand[BestCand].PhysReg, TRI) << '\n';
1445  });
1446  continue;
1447  }
1448  growRegion(Cand);
1449 
1450  SpillPlacer->finish();
1451 
1452  // No live bundles, defer to splitSingleBlocks().
1453  if (!Cand.LiveBundles.any()) {
1454  DEBUG(dbgs() << " no bundles.\n");
1455  continue;
1456  }
1457 
1458  Cost += calcGlobalSplitCost(Cand);
1459  DEBUG({
1460  dbgs() << ", total = "; MBFI->printBlockFreq(dbgs(), Cost)
1461  << " with bundles";
1462  for (int i = Cand.LiveBundles.find_first(); i>=0;
1463  i = Cand.LiveBundles.find_next(i))
1464  dbgs() << " EB#" << i;
1465  dbgs() << ".\n";
1466  });
1467  if (Cost < BestCost) {
1468  BestCand = NumCands;
1469  BestCost = Cost;
1470  }
1471  ++NumCands;
1472  }
1473  return BestCand;
1474 }
1475 
1476 unsigned RAGreedy::doRegionSplit(LiveInterval &VirtReg, unsigned BestCand,
1477  bool HasCompact,
1478  SmallVectorImpl<unsigned> &NewVRegs) {
1479  SmallVector<unsigned, 8> UsedCands;
1480  // Prepare split editor.
1481  LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
1482  SE->reset(LREdit, SplitSpillMode);
1483 
1484  // Assign all edge bundles to the preferred candidate, or NoCand.
1485  BundleCand.assign(Bundles->getNumBundles(), NoCand);
1486 
1487  // Assign bundles for the best candidate region.
1488  if (BestCand != NoCand) {
1489  GlobalSplitCandidate &Cand = GlobalCand[BestCand];
1490  if (unsigned B = Cand.getBundles(BundleCand, BestCand)) {
1491  UsedCands.push_back(BestCand);
1492  Cand.IntvIdx = SE->openIntv();
1493  DEBUG(dbgs() << "Split for " << PrintReg(Cand.PhysReg, TRI) << " in "
1494  << B << " bundles, intv " << Cand.IntvIdx << ".\n");
1495  (void)B;
1496  }
1497  }
1498 
1499  // Assign bundles for the compact region.
1500  if (HasCompact) {
1501  GlobalSplitCandidate &Cand = GlobalCand.front();
1502  assert(!Cand.PhysReg && "Compact region has no physreg");
1503  if (unsigned B = Cand.getBundles(BundleCand, 0)) {
1504  UsedCands.push_back(0);
1505  Cand.IntvIdx = SE->openIntv();
1506  DEBUG(dbgs() << "Split for compact region in " << B << " bundles, intv "
1507  << Cand.IntvIdx << ".\n");
1508  (void)B;
1509  }
1510  }
1511 
1512  splitAroundRegion(LREdit, UsedCands);
1513  return 0;
1514 }
1515 
1516 
1517 //===----------------------------------------------------------------------===//
1518 // Per-Block Splitting
1519 //===----------------------------------------------------------------------===//
1520 
1521 /// tryBlockSplit - Split a global live range around every block with uses. This
1522 /// creates a lot of local live ranges, that will be split by tryLocalSplit if
1523 /// they don't allocate.
1524 unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order,
1525  SmallVectorImpl<unsigned> &NewVRegs) {
1526  assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed");
1527  unsigned Reg = VirtReg.reg;
1528  bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
1529  LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
1530  SE->reset(LREdit, SplitSpillMode);
1531  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
1532  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
1533  const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
1534  if (SA->shouldSplitSingleBlock(BI, SingleInstrs))
1535  SE->splitSingleBlock(BI);
1536  }
1537  // No blocks were split.
1538  if (LREdit.empty())
1539  return 0;
1540 
1541  // We did split for some blocks.
1542  SmallVector<unsigned, 8> IntvMap;
1543  SE->finish(&IntvMap);
1544 
1545  // Tell LiveDebugVariables about the new ranges.
1546  DebugVars->splitRegister(Reg, LREdit.regs(), *LIS);
1547 
1548  ExtraRegInfo.resize(MRI->getNumVirtRegs());
1549 
1550  // Sort out the new intervals created by splitting. The remainder interval
1551  // goes straight to spilling, the new local ranges get to stay RS_New.
1552  for (unsigned i = 0, e = LREdit.size(); i != e; ++i) {
1553  LiveInterval &LI = LIS->getInterval(LREdit.get(i));
1554  if (getStage(LI) == RS_New && IntvMap[i] == 0)
1555  setStage(LI, RS_Spill);
1556  }
1557 
1558  if (VerifyEnabled)
1559  MF->verify(this, "After splitting live range around basic blocks");
1560  return 0;
1561 }
1562 
1563 
1564 //===----------------------------------------------------------------------===//
1565 // Per-Instruction Splitting
1566 //===----------------------------------------------------------------------===//
1567 
1568 /// Get the number of allocatable registers that match the constraints of \p Reg
1569 /// on \p MI and that are also in \p SuperRC.
1571  const MachineInstr *MI, unsigned Reg, const TargetRegisterClass *SuperRC,
1572  const TargetInstrInfo *TII, const TargetRegisterInfo *TRI,
1573  const RegisterClassInfo &RCI) {
1574  assert(SuperRC && "Invalid register class");
1575 
1576  const TargetRegisterClass *ConstrainedRC =
1577  MI->getRegClassConstraintEffectForVReg(Reg, SuperRC, TII, TRI,
1578  /* ExploreBundle */ true);
1579  if (!ConstrainedRC)
1580  return 0;
1581  return RCI.getNumAllocatableRegs(ConstrainedRC);
1582 }
1583 
1584 /// tryInstructionSplit - Split a live range around individual instructions.
1585 /// This is normally not worthwhile since the spiller is doing essentially the
1586 /// same thing. However, when the live range is in a constrained register
1587 /// class, it may help to insert copies such that parts of the live range can
1588 /// be moved to a larger register class.
1589 ///
1590 /// This is similar to spilling to a larger register class.
1591 unsigned
1592 RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
1593  SmallVectorImpl<unsigned> &NewVRegs) {
1594  const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg);
1595  // There is no point to this if there are no larger sub-classes.
1596  if (!RegClassInfo.isProperSubClass(CurRC))
1597  return 0;
1598 
1599  // Always enable split spill mode, since we're effectively spilling to a
1600  // register.
1601  LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
1602  SE->reset(LREdit, SplitEditor::SM_Size);
1603 
1604  ArrayRef<SlotIndex> Uses = SA->getUseSlots();
1605  if (Uses.size() <= 1)
1606  return 0;
1607 
1608  DEBUG(dbgs() << "Split around " << Uses.size() << " individual instrs.\n");
1609 
1610  const TargetRegisterClass *SuperRC =
1611  TRI->getLargestLegalSuperClass(CurRC, *MF);
1612  unsigned SuperRCNumAllocatableRegs = RCI.getNumAllocatableRegs(SuperRC);
1613  // Split around every non-copy instruction if this split will relax
1614  // the constraints on the virtual register.
1615  // Otherwise, splitting just inserts uncoalescable copies that do not help
1616  // the allocation.
1617  for (unsigned i = 0; i != Uses.size(); ++i) {
1618  if (const MachineInstr *MI = Indexes->getInstructionFromIndex(Uses[i]))
1619  if (MI->isFullCopy() ||
1620  SuperRCNumAllocatableRegs ==
1621  getNumAllocatableRegsForConstraints(MI, VirtReg.reg, SuperRC, TII,
1622  TRI, RCI)) {
1623  DEBUG(dbgs() << " skip:\t" << Uses[i] << '\t' << *MI);
1624  continue;
1625  }
1626  SE->openIntv();
1627  SlotIndex SegStart = SE->enterIntvBefore(Uses[i]);
1628  SlotIndex SegStop = SE->leaveIntvAfter(Uses[i]);
1629  SE->useIntv(SegStart, SegStop);
1630  }
1631 
1632  if (LREdit.empty()) {
1633  DEBUG(dbgs() << "All uses were copies.\n");
1634  return 0;
1635  }
1636 
1637  SmallVector<unsigned, 8> IntvMap;
1638  SE->finish(&IntvMap);
1639  DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS);
1640  ExtraRegInfo.resize(MRI->getNumVirtRegs());
1641 
1642  // Assign all new registers to RS_Spill. This was the last chance.
1643  setStage(LREdit.begin(), LREdit.end(), RS_Spill);
1644  return 0;
1645 }
1646 
1647 
1648 //===----------------------------------------------------------------------===//
1649 // Local Splitting
1650 //===----------------------------------------------------------------------===//
1651 
1652 
1653 /// calcGapWeights - Compute the maximum spill weight that needs to be evicted
1654 /// in order to use PhysReg between two entries in SA->UseSlots.
1655 ///
1656 /// GapWeight[i] represents the gap between UseSlots[i] and UseSlots[i+1].
1657 ///
1658 void RAGreedy::calcGapWeights(unsigned PhysReg,
1659  SmallVectorImpl<float> &GapWeight) {
1660  assert(SA->getUseBlocks().size() == 1 && "Not a local interval");
1661  const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front();
1662  ArrayRef<SlotIndex> Uses = SA->getUseSlots();
1663  const unsigned NumGaps = Uses.size()-1;
1664 
1665  // Start and end points for the interference check.
1666  SlotIndex StartIdx =
1667  BI.LiveIn ? BI.FirstInstr.getBaseIndex() : BI.FirstInstr;
1668  SlotIndex StopIdx =
1669  BI.LiveOut ? BI.LastInstr.getBoundaryIndex() : BI.LastInstr;
1670 
1671  GapWeight.assign(NumGaps, 0.0f);
1672 
1673  // Add interference from each overlapping register.
1674  for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
1675  if (!Matrix->query(const_cast<LiveInterval&>(SA->getParent()), *Units)
1676  .checkInterference())
1677  continue;
1678 
1679  // We know that VirtReg is a continuous interval from FirstInstr to
1680  // LastInstr, so we don't need InterferenceQuery.
1681  //
1682  // Interference that overlaps an instruction is counted in both gaps
1683  // surrounding the instruction. The exception is interference before
1684  // StartIdx and after StopIdx.
1685  //
1687  Matrix->getLiveUnions()[*Units] .find(StartIdx);
1688  for (unsigned Gap = 0; IntI.valid() && IntI.start() < StopIdx; ++IntI) {
1689  // Skip the gaps before IntI.
1690  while (Uses[Gap+1].getBoundaryIndex() < IntI.start())
1691  if (++Gap == NumGaps)
1692  break;
1693  if (Gap == NumGaps)
1694  break;
1695 
1696  // Update the gaps covered by IntI.
1697  const float weight = IntI.value()->weight;
1698  for (; Gap != NumGaps; ++Gap) {
1699  GapWeight[Gap] = std::max(GapWeight[Gap], weight);
1700  if (Uses[Gap+1].getBaseIndex() >= IntI.stop())
1701  break;
1702  }
1703  if (Gap == NumGaps)
1704  break;
1705  }
1706  }
1707 
1708  // Add fixed interference.
1709  for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
1710  const LiveRange &LR = LIS->getRegUnit(*Units);
1711  LiveRange::const_iterator I = LR.find(StartIdx);
1713 
1714  // Same loop as above. Mark any overlapped gaps as HUGE_VALF.
1715  for (unsigned Gap = 0; I != E && I->start < StopIdx; ++I) {
1716  while (Uses[Gap+1].getBoundaryIndex() < I->start)
1717  if (++Gap == NumGaps)
1718  break;
1719  if (Gap == NumGaps)
1720  break;
1721 
1722  for (; Gap != NumGaps; ++Gap) {
1723  GapWeight[Gap] = llvm::huge_valf;
1724  if (Uses[Gap+1].getBaseIndex() >= I->end)
1725  break;
1726  }
1727  if (Gap == NumGaps)
1728  break;
1729  }
1730  }
1731 }
1732 
1733 /// tryLocalSplit - Try to split VirtReg into smaller intervals inside its only
1734 /// basic block.
1735 ///
1736 unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
1737  SmallVectorImpl<unsigned> &NewVRegs) {
1738  assert(SA->getUseBlocks().size() == 1 && "Not a local interval");
1739  const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front();
1740 
1741  // Note that it is possible to have an interval that is live-in or live-out
1742  // while only covering a single block - A phi-def can use undef values from
1743  // predecessors, and the block could be a single-block loop.
1744  // We don't bother doing anything clever about such a case, we simply assume
1745  // that the interval is continuous from FirstInstr to LastInstr. We should
1746  // make sure that we don't do anything illegal to such an interval, though.
1747 
1748  ArrayRef<SlotIndex> Uses = SA->getUseSlots();
1749  if (Uses.size() <= 2)
1750  return 0;
1751  const unsigned NumGaps = Uses.size()-1;
1752 
1753  DEBUG({
1754  dbgs() << "tryLocalSplit: ";
1755  for (unsigned i = 0, e = Uses.size(); i != e; ++i)
1756  dbgs() << ' ' << Uses[i];
1757  dbgs() << '\n';
1758  });
1759 
1760  // If VirtReg is live across any register mask operands, compute a list of
1761  // gaps with register masks.
1762  SmallVector<unsigned, 8> RegMaskGaps;
1763  if (Matrix->checkRegMaskInterference(VirtReg)) {
1764  // Get regmask slots for the whole block.
1765  ArrayRef<SlotIndex> RMS = LIS->getRegMaskSlotsInBlock(BI.MBB->getNumber());
1766  DEBUG(dbgs() << RMS.size() << " regmasks in block:");
1767  // Constrain to VirtReg's live range.
1768  unsigned ri = std::lower_bound(RMS.begin(), RMS.end(),
1769  Uses.front().getRegSlot()) - RMS.begin();
1770  unsigned re = RMS.size();
1771  for (unsigned i = 0; i != NumGaps && ri != re; ++i) {
1772  // Look for Uses[i] <= RMS <= Uses[i+1].
1773  assert(!SlotIndex::isEarlierInstr(RMS[ri], Uses[i]));
1774  if (SlotIndex::isEarlierInstr(Uses[i+1], RMS[ri]))
1775  continue;
1776  // Skip a regmask on the same instruction as the last use. It doesn't
1777  // overlap the live range.
1778  if (SlotIndex::isSameInstr(Uses[i+1], RMS[ri]) && i+1 == NumGaps)
1779  break;
1780  DEBUG(dbgs() << ' ' << RMS[ri] << ':' << Uses[i] << '-' << Uses[i+1]);
1781  RegMaskGaps.push_back(i);
1782  // Advance ri to the next gap. A regmask on one of the uses counts in
1783  // both gaps.
1784  while (ri != re && SlotIndex::isEarlierInstr(RMS[ri], Uses[i+1]))
1785  ++ri;
1786  }
1787  DEBUG(dbgs() << '\n');
1788  }
1789 
1790  // Since we allow local split results to be split again, there is a risk of
1791  // creating infinite loops. It is tempting to require that the new live
1792  // ranges have less instructions than the original. That would guarantee
1793  // convergence, but it is too strict. A live range with 3 instructions can be
1794  // split 2+3 (including the COPY), and we want to allow that.
1795  //
1796  // Instead we use these rules:
1797  //
1798  // 1. Allow any split for ranges with getStage() < RS_Split2. (Except for the
1799  // noop split, of course).
1800  // 2. Require progress be made for ranges with getStage() == RS_Split2. All
1801  // the new ranges must have fewer instructions than before the split.
1802  // 3. New ranges with the same number of instructions are marked RS_Split2,
1803  // smaller ranges are marked RS_New.
1804  //
1805  // These rules allow a 3 -> 2+3 split once, which we need. They also prevent
1806  // excessive splitting and infinite loops.
1807  //
1808  bool ProgressRequired = getStage(VirtReg) >= RS_Split2;
1809 
1810  // Best split candidate.
1811  unsigned BestBefore = NumGaps;
1812  unsigned BestAfter = 0;
1813  float BestDiff = 0;
1814 
1815  const float blockFreq =
1816  SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() *
1817  (1.0f / MBFI->getEntryFreq());
1818  SmallVector<float, 8> GapWeight;
1819 
1820  Order.rewind();
1821  while (unsigned PhysReg = Order.next()) {
1822  // Keep track of the largest spill weight that would need to be evicted in
1823  // order to make use of PhysReg between UseSlots[i] and UseSlots[i+1].
1824  calcGapWeights(PhysReg, GapWeight);
1825 
1826  // Remove any gaps with regmask clobbers.
1827  if (Matrix->checkRegMaskInterference(VirtReg, PhysReg))
1828  for (unsigned i = 0, e = RegMaskGaps.size(); i != e; ++i)
1829  GapWeight[RegMaskGaps[i]] = llvm::huge_valf;
1830 
1831  // Try to find the best sequence of gaps to close.
1832  // The new spill weight must be larger than any gap interference.
1833 
1834  // We will split before Uses[SplitBefore] and after Uses[SplitAfter].
1835  unsigned SplitBefore = 0, SplitAfter = 1;
1836 
1837  // MaxGap should always be max(GapWeight[SplitBefore..SplitAfter-1]).
1838  // It is the spill weight that needs to be evicted.
1839  float MaxGap = GapWeight[0];
1840 
1841  for (;;) {
1842  // Live before/after split?
1843  const bool LiveBefore = SplitBefore != 0 || BI.LiveIn;
1844  const bool LiveAfter = SplitAfter != NumGaps || BI.LiveOut;
1845 
1846  DEBUG(dbgs() << PrintReg(PhysReg, TRI) << ' '
1847  << Uses[SplitBefore] << '-' << Uses[SplitAfter]
1848  << " i=" << MaxGap);
1849 
1850  // Stop before the interval gets so big we wouldn't be making progress.
1851  if (!LiveBefore && !LiveAfter) {
1852  DEBUG(dbgs() << " all\n");
1853  break;
1854  }
1855  // Should the interval be extended or shrunk?
1856  bool Shrink = true;
1857 
1858  // How many gaps would the new range have?
1859  unsigned NewGaps = LiveBefore + SplitAfter - SplitBefore + LiveAfter;
1860 
1861  // Legally, without causing looping?
1862  bool Legal = !ProgressRequired || NewGaps < NumGaps;
1863 
1864  if (Legal && MaxGap < llvm::huge_valf) {
1865  // Estimate the new spill weight. Each instruction reads or writes the
1866  // register. Conservatively assume there are no read-modify-write
1867  // instructions.
1868  //
1869  // Try to guess the size of the new interval.
1870  const float EstWeight = normalizeSpillWeight(
1871  blockFreq * (NewGaps + 1),
1872  Uses[SplitBefore].distance(Uses[SplitAfter]) +
1873  (LiveBefore + LiveAfter) * SlotIndex::InstrDist,
1874  1);
1875  // Would this split be possible to allocate?
1876  // Never allocate all gaps, we wouldn't be making progress.
1877  DEBUG(dbgs() << " w=" << EstWeight);
1878  if (EstWeight * Hysteresis >= MaxGap) {
1879  Shrink = false;
1880  float Diff = EstWeight - MaxGap;
1881  if (Diff > BestDiff) {
1882  DEBUG(dbgs() << " (best)");
1883  BestDiff = Hysteresis * Diff;
1884  BestBefore = SplitBefore;
1885  BestAfter = SplitAfter;
1886  }
1887  }
1888  }
1889 
1890  // Try to shrink.
1891  if (Shrink) {
1892  if (++SplitBefore < SplitAfter) {
1893  DEBUG(dbgs() << " shrink\n");
1894  // Recompute the max when necessary.
1895  if (GapWeight[SplitBefore - 1] >= MaxGap) {
1896  MaxGap = GapWeight[SplitBefore];
1897  for (unsigned i = SplitBefore + 1; i != SplitAfter; ++i)
1898  MaxGap = std::max(MaxGap, GapWeight[i]);
1899  }
1900  continue;
1901  }
1902  MaxGap = 0;
1903  }
1904 
1905  // Try to extend the interval.
1906  if (SplitAfter >= NumGaps) {
1907  DEBUG(dbgs() << " end\n");
1908  break;
1909  }
1910 
1911  DEBUG(dbgs() << " extend\n");
1912  MaxGap = std::max(MaxGap, GapWeight[SplitAfter++]);
1913  }
1914  }
1915 
1916  // Didn't find any candidates?
1917  if (BestBefore == NumGaps)
1918  return 0;
1919 
1920  DEBUG(dbgs() << "Best local split range: " << Uses[BestBefore]
1921  << '-' << Uses[BestAfter] << ", " << BestDiff
1922  << ", " << (BestAfter - BestBefore + 1) << " instrs\n");
1923 
1924  LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
1925  SE->reset(LREdit);
1926 
1927  SE->openIntv();
1928  SlotIndex SegStart = SE->enterIntvBefore(Uses[BestBefore]);
1929  SlotIndex SegStop = SE->leaveIntvAfter(Uses[BestAfter]);
1930  SE->useIntv(SegStart, SegStop);
1931  SmallVector<unsigned, 8> IntvMap;
1932  SE->finish(&IntvMap);
1933  DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS);
1934 
1935  // If the new range has the same number of instructions as before, mark it as
1936  // RS_Split2 so the next split will be forced to make progress. Otherwise,
1937  // leave the new intervals as RS_New so they can compete.
1938  bool LiveBefore = BestBefore != 0 || BI.LiveIn;
1939  bool LiveAfter = BestAfter != NumGaps || BI.LiveOut;
1940  unsigned NewGaps = LiveBefore + BestAfter - BestBefore + LiveAfter;
1941  if (NewGaps >= NumGaps) {
1942  DEBUG(dbgs() << "Tagging non-progress ranges: ");
1943  assert(!ProgressRequired && "Didn't make progress when it was required.");
1944  for (unsigned i = 0, e = IntvMap.size(); i != e; ++i)
1945  if (IntvMap[i] == 1) {
1946  setStage(LIS->getInterval(LREdit.get(i)), RS_Split2);
1947  DEBUG(dbgs() << PrintReg(LREdit.get(i)));
1948  }
1949  DEBUG(dbgs() << '\n');
1950  }
1951  ++NumLocalSplits;
1952 
1953  return 0;
1954 }
1955 
1956 //===----------------------------------------------------------------------===//
1957 // Live Range Splitting
1958 //===----------------------------------------------------------------------===//
1959 
1960 /// trySplit - Try to split VirtReg or one of its interferences, making it
1961 /// assignable.
1962 /// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
1963 unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
1964  SmallVectorImpl<unsigned>&NewVRegs) {
1965  // Ranges must be Split2 or less.
1966  if (getStage(VirtReg) >= RS_Spill)
1967  return 0;
1968 
1969  // Local intervals are handled separately.
1970  if (LIS->intervalIsInOneMBB(VirtReg)) {
1971  NamedRegionTimer T("local_split", "Local Splitting", TimerGroupName,
1972  TimerGroupDescription, TimePassesIsEnabled);
1973  SA->analyze(&VirtReg);
1974  unsigned PhysReg = tryLocalSplit(VirtReg, Order, NewVRegs);
1975  if (PhysReg || !NewVRegs.empty())
1976  return PhysReg;
1977  return tryInstructionSplit(VirtReg, Order, NewVRegs);
1978  }
1979 
1980  NamedRegionTimer T("global_split", "Global Splitting", TimerGroupName,
1981  TimerGroupDescription, TimePassesIsEnabled);
1982 
1983  SA->analyze(&VirtReg);
1984 
1985  // FIXME: SplitAnalysis may repair broken live ranges coming from the
1986  // coalescer. That may cause the range to become allocatable which means that
1987  // tryRegionSplit won't be making progress. This check should be replaced with
1988  // an assertion when the coalescer is fixed.
1989  if (SA->didRepairRange()) {
1990  // VirtReg has changed, so all cached queries are invalid.
1991  Matrix->invalidateVirtRegs();
1992  if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs))
1993  return PhysReg;
1994  }
1995 
1996  // First try to split around a region spanning multiple blocks. RS_Split2
1997  // ranges already made dubious progress with region splitting, so they go
1998  // straight to single block splitting.
1999  if (getStage(VirtReg) < RS_Split2) {
2000  unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
2001  if (PhysReg || !NewVRegs.empty())
2002  return PhysReg;
2003  }
2004 
2005  // Then isolate blocks.
2006  return tryBlockSplit(VirtReg, Order, NewVRegs);
2007 }
2008 
2009 //===----------------------------------------------------------------------===//
2010 // Last Chance Recoloring
2011 //===----------------------------------------------------------------------===//
2012 
2013 /// mayRecolorAllInterferences - Check if the virtual registers that
2014 /// interfere with \p VirtReg on \p PhysReg (or one of its aliases) may be
2015 /// recolored to free \p PhysReg.
2016 /// When true is returned, \p RecoloringCandidates has been augmented with all
2017 /// the live intervals that need to be recolored in order to free \p PhysReg
2018 /// for \p VirtReg.
2019 /// \p FixedRegisters contains all the virtual registers that cannot be
2020 /// recolored.
2021 bool
2022 RAGreedy::mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg,
2023  SmallLISet &RecoloringCandidates,
2024  const SmallVirtRegSet &FixedRegisters) {
2025  const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg);
2026 
2027  for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
2028  LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units);
2029  // If there is LastChanceRecoloringMaxInterference or more interferences,
2030  // chances are one would not be recolorable.
2033  DEBUG(dbgs() << "Early abort: too many interferences.\n");
2034  CutOffInfo |= CO_Interf;
2035  return false;
2036  }
2037  for (unsigned i = Q.interferingVRegs().size(); i; --i) {
2038  LiveInterval *Intf = Q.interferingVRegs()[i - 1];
2039  // If Intf is done and sit on the same register class as VirtReg,
2040  // it would not be recolorable as it is in the same state as VirtReg.
2041  if ((getStage(*Intf) == RS_Done &&
2042  MRI->getRegClass(Intf->reg) == CurRC) ||
2043  FixedRegisters.count(Intf->reg)) {
2044  DEBUG(dbgs() << "Early abort: the inteference is not recolorable.\n");
2045  return false;
2046  }
2047  RecoloringCandidates.insert(Intf);
2048  }
2049  }
2050  return true;
2051 }
2052 
2053 /// tryLastChanceRecoloring - Try to assign a color to \p VirtReg by recoloring
2054 /// its interferences.
2055 /// Last chance recoloring chooses a color for \p VirtReg and recolors every
2056 /// virtual register that was using it. The recoloring process may recursively
2057 /// use the last chance recoloring. Therefore, when a virtual register has been
2058 /// assigned a color by this mechanism, it is marked as Fixed, i.e., it cannot
2059 /// be last-chance-recolored again during this recoloring "session".
2060 /// E.g.,
2061 /// Let
2062 /// vA can use {R1, R2 }
2063 /// vB can use { R2, R3}
2064 /// vC can use {R1 }
2065 /// Where vA, vB, and vC cannot be split anymore (they are reloads for
2066 /// instance) and they all interfere.
2067 ///
2068 /// vA is assigned R1
2069 /// vB is assigned R2
2070 /// vC tries to evict vA but vA is already done.
2071 /// Regular register allocation fails.
2072 ///
2073 /// Last chance recoloring kicks in:
2074 /// vC does as if vA was evicted => vC uses R1.
2075 /// vC is marked as fixed.
2076 /// vA needs to find a color.
2077 /// None are available.
2078 /// vA cannot evict vC: vC is a fixed virtual register now.
2079 /// vA does as if vB was evicted => vA uses R2.
2080 /// vB needs to find a color.
2081 /// R3 is available.
2082 /// Recoloring => vC = R1, vA = R2, vB = R3
2083 ///
2084 /// \p Order defines the preferred allocation order for \p VirtReg.
2085 /// \p NewRegs will contain any new virtual register that have been created
2086 /// (split, spill) during the process and that must be assigned.
2087 /// \p FixedRegisters contains all the virtual registers that cannot be
2088 /// recolored.
2089 /// \p Depth gives the current depth of the last chance recoloring.
2090 /// \return a physical register that can be used for VirtReg or ~0u if none
2091 /// exists.
2092 unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
2093  AllocationOrder &Order,
2094  SmallVectorImpl<unsigned> &NewVRegs,
2095  SmallVirtRegSet &FixedRegisters,
2096  unsigned Depth) {
2097  DEBUG(dbgs() << "Try last chance recoloring for " << VirtReg << '\n');
2098  // Ranges must be Done.
2099  assert((getStage(VirtReg) >= RS_Done || !VirtReg.isSpillable()) &&
2100  "Last chance recoloring should really be last chance");
2101  // Set the max depth to LastChanceRecoloringMaxDepth.
2102  // We may want to reconsider that if we end up with a too large search space
2103  // for target with hundreds of registers.
2104  // Indeed, in that case we may want to cut the search space earlier.
2106  DEBUG(dbgs() << "Abort because max depth has been reached.\n");
2107  CutOffInfo |= CO_Depth;
2108  return ~0u;
2109  }
2110 
2111  // Set of Live intervals that will need to be recolored.
2112  SmallLISet RecoloringCandidates;
2113  // Record the original mapping virtual register to physical register in case
2114  // the recoloring fails.
2115  DenseMap<unsigned, unsigned> VirtRegToPhysReg;
2116  // Mark VirtReg as fixed, i.e., it will not be recolored pass this point in
2117  // this recoloring "session".
2118  FixedRegisters.insert(VirtReg.reg);
2119  SmallVector<unsigned, 4> CurrentNewVRegs;
2120 
2121  Order.rewind();
2122  while (unsigned PhysReg = Order.next()) {
2123  DEBUG(dbgs() << "Try to assign: " << VirtReg << " to "
2124  << PrintReg(PhysReg, TRI) << '\n');
2125  RecoloringCandidates.clear();
2126  VirtRegToPhysReg.clear();
2127  CurrentNewVRegs.clear();
2128 
2129  // It is only possible to recolor virtual register interference.
2130  if (Matrix->checkInterference(VirtReg, PhysReg) >
2131  LiveRegMatrix::IK_VirtReg) {
2132  DEBUG(dbgs() << "Some inteferences are not with virtual registers.\n");
2133 
2134  continue;
2135  }
2136 
2137  // Early give up on this PhysReg if it is obvious we cannot recolor all
2138  // the interferences.
2139  if (!mayRecolorAllInterferences(PhysReg, VirtReg, RecoloringCandidates,
2140  FixedRegisters)) {
2141  DEBUG(dbgs() << "Some inteferences cannot be recolored.\n");
2142  continue;
2143  }
2144 
2145  // RecoloringCandidates contains all the virtual registers that interfer
2146  // with VirtReg on PhysReg (or one of its aliases).
2147  // Enqueue them for recoloring and perform the actual recoloring.
2148  PQueue RecoloringQueue;
2149  for (SmallLISet::iterator It = RecoloringCandidates.begin(),
2150  EndIt = RecoloringCandidates.end();
2151  It != EndIt; ++It) {
2152  unsigned ItVirtReg = (*It)->reg;
2153  enqueue(RecoloringQueue, *It);
2154  assert(VRM->hasPhys(ItVirtReg) &&
2155  "Interferences are supposed to be with allocated vairables");
2156 
2157  // Record the current allocation.
2158  VirtRegToPhysReg[ItVirtReg] = VRM->getPhys(ItVirtReg);
2159  // unset the related struct.
2160  Matrix->unassign(**It);
2161  }
2162 
2163  // Do as if VirtReg was assigned to PhysReg so that the underlying
2164  // recoloring has the right information about the interferes and
2165  // available colors.
2166  Matrix->assign(VirtReg, PhysReg);
2167 
2168  // Save the current recoloring state.
2169  // If we cannot recolor all the interferences, we will have to start again
2170  // at this point for the next physical register.
2171  SmallVirtRegSet SaveFixedRegisters(FixedRegisters);
2172  if (tryRecoloringCandidates(RecoloringQueue, CurrentNewVRegs,
2173  FixedRegisters, Depth)) {
2174  // Push the queued vregs into the main queue.
2175  for (unsigned NewVReg : CurrentNewVRegs)
2176  NewVRegs.push_back(NewVReg);
2177  // Do not mess up with the global assignment process.
2178  // I.e., VirtReg must be unassigned.
2179  Matrix->unassign(VirtReg);
2180  return PhysReg;
2181  }
2182 
2183  DEBUG(dbgs() << "Fail to assign: " << VirtReg << " to "
2184  << PrintReg(PhysReg, TRI) << '\n');
2185 
2186  // The recoloring attempt failed, undo the changes.
2187  FixedRegisters = SaveFixedRegisters;
2188  Matrix->unassign(VirtReg);
2189 
2190  // For a newly created vreg which is also in RecoloringCandidates,
2191  // don't add it to NewVRegs because its physical register will be restored
2192  // below. Other vregs in CurrentNewVRegs are created by calling
2193  // selectOrSplit and should be added into NewVRegs.
2194  for (SmallVectorImpl<unsigned>::iterator Next = CurrentNewVRegs.begin(),
2195  End = CurrentNewVRegs.end();
2196  Next != End; ++Next) {
2197  if (RecoloringCandidates.count(&LIS->getInterval(*Next)))
2198  continue;
2199  NewVRegs.push_back(*Next);
2200  }
2201 
2202  for (SmallLISet::iterator It = RecoloringCandidates.begin(),
2203  EndIt = RecoloringCandidates.end();
2204  It != EndIt; ++It) {
2205  unsigned ItVirtReg = (*It)->reg;
2206  if (VRM->hasPhys(ItVirtReg))
2207  Matrix->unassign(**It);
2208  unsigned ItPhysReg = VirtRegToPhysReg[ItVirtReg];
2209  Matrix->assign(**It, ItPhysReg);
2210  }
2211  }
2212 
2213  // Last chance recoloring did not worked either, give up.
2214  return ~0u;
2215 }
2216 
2217 /// tryRecoloringCandidates - Try to assign a new color to every register
2218 /// in \RecoloringQueue.
2219 /// \p NewRegs will contain any new virtual register created during the
2220 /// recoloring process.
2221 /// \p FixedRegisters[in/out] contains all the registers that have been
2222 /// recolored.
2223 /// \return true if all virtual registers in RecoloringQueue were successfully
2224 /// recolored, false otherwise.
2225 bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue,
2226  SmallVectorImpl<unsigned> &NewVRegs,
2227  SmallVirtRegSet &FixedRegisters,
2228  unsigned Depth) {
2229  while (!RecoloringQueue.empty()) {
2230  LiveInterval *LI = dequeue(RecoloringQueue);
2231  DEBUG(dbgs() << "Try to recolor: " << *LI << '\n');
2232  unsigned PhysReg;
2233  PhysReg = selectOrSplitImpl(*LI, NewVRegs, FixedRegisters, Depth + 1);
2234  // When splitting happens, the live-range may actually be empty.
2235  // In that case, this is okay to continue the recoloring even
2236  // if we did not find an alternative color for it. Indeed,
2237  // there will not be anything to color for LI in the end.
2238  if (PhysReg == ~0u || (!PhysReg && !LI->empty()))
2239  return false;
2240 
2241  if (!PhysReg) {
2242  assert(LI->empty() && "Only empty live-range do not require a register");
2243  DEBUG(dbgs() << "Recoloring of " << *LI << " succeeded. Empty LI.\n");
2244  continue;
2245  }
2246  DEBUG(dbgs() << "Recoloring of " << *LI
2247  << " succeeded with: " << PrintReg(PhysReg, TRI) << '\n');
2248 
2249  Matrix->assign(*LI, PhysReg);
2250  FixedRegisters.insert(LI->reg);
2251  }
2252  return true;
2253 }
2254 
2255 //===----------------------------------------------------------------------===//
2256 // Main Entry Point
2257 //===----------------------------------------------------------------------===//
2258 
2259 unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
2260  SmallVectorImpl<unsigned> &NewVRegs) {
2261  CutOffInfo = CO_None;
2262  LLVMContext &Ctx = MF->getFunction()->getContext();
2263  SmallVirtRegSet FixedRegisters;
2264  unsigned Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters);
2265  if (Reg == ~0U && (CutOffInfo != CO_None)) {
2266  uint8_t CutOffEncountered = CutOffInfo & (CO_Depth | CO_Interf);
2267  if (CutOffEncountered == CO_Depth)
2268  Ctx.emitError("register allocation failed: maximum depth for recoloring "
2269  "reached. Use -fexhaustive-register-search to skip "
2270  "cutoffs");
2271  else if (CutOffEncountered == CO_Interf)
2272  Ctx.emitError("register allocation failed: maximum interference for "
2273  "recoloring reached. Use -fexhaustive-register-search "
2274  "to skip cutoffs");
2275  else if (CutOffEncountered == (CO_Depth | CO_Interf))
2276  Ctx.emitError("register allocation failed: maximum interference and "
2277  "depth for recoloring reached. Use "
2278  "-fexhaustive-register-search to skip cutoffs");
2279  }
2280  return Reg;
2281 }
2282 
2283 /// Using a CSR for the first time has a cost because it causes push|pop
2284 /// to be added to prologue|epilogue. Splitting a cold section of the live
2285 /// range can have lower cost than using the CSR for the first time;
2286 /// Spilling a live range in the cold path can have lower cost than using
2287 /// the CSR for the first time. Returns the physical register if we decide
2288 /// to use the CSR; otherwise return 0.
2289 unsigned RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg,
2290  AllocationOrder &Order,
2291  unsigned PhysReg,
2292  unsigned &CostPerUseLimit,
2293  SmallVectorImpl<unsigned> &NewVRegs) {
2294  if (getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) {
2295  // We choose spill over using the CSR for the first time if the spill cost
2296  // is lower than CSRCost.
2297  SA->analyze(&VirtReg);
2298  if (calcSpillCost() >= CSRCost)
2299  return PhysReg;
2300 
2301  // We are going to spill, set CostPerUseLimit to 1 to make sure that
2302  // we will not use a callee-saved register in tryEvict.
2303  CostPerUseLimit = 1;
2304  return 0;
2305  }
2306  if (getStage(VirtReg) < RS_Split) {
2307  // We choose pre-splitting over using the CSR for the first time if
2308  // the cost of splitting is lower than CSRCost.
2309  SA->analyze(&VirtReg);
2310  unsigned NumCands = 0;
2311  BlockFrequency BestCost = CSRCost; // Don't modify CSRCost.
2312  unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, BestCost,
2313  NumCands, true /*IgnoreCSR*/);
2314  if (BestCand == NoCand)
2315  // Use the CSR if we can't find a region split below CSRCost.
2316  return PhysReg;
2317 
2318  // Perform the actual pre-splitting.
2319  doRegionSplit(VirtReg, BestCand, false/*HasCompact*/, NewVRegs);
2320  return 0;
2321  }
2322  return PhysReg;
2323 }
2324 
2325 void RAGreedy::aboutToRemoveInterval(LiveInterval &LI) {
2326  // Do not keep invalid information around.
2327  SetOfBrokenHints.remove(&LI);
2328 }
2329 
2330 void RAGreedy::initializeCSRCost() {
2331  // We use the larger one out of the command-line option and the value report
2332  // by TRI.
2333  CSRCost = BlockFrequency(
2334  std::max((unsigned)CSRFirstTimeCost, TRI->getCSRFirstUseCost()));
2335  if (!CSRCost.getFrequency())
2336  return;
2337 
2338  // Raw cost is relative to Entry == 2^14; scale it appropriately.
2339  uint64_t ActualEntry = MBFI->getEntryFreq();
2340  if (!ActualEntry) {
2341  CSRCost = 0;
2342  return;
2343  }
2344  uint64_t FixedEntry = 1 << 14;
2345  if (ActualEntry < FixedEntry)
2346  CSRCost *= BranchProbability(ActualEntry, FixedEntry);
2347  else if (ActualEntry <= UINT32_MAX)
2348  // Invert the fraction and divide.
2349  CSRCost /= BranchProbability(FixedEntry, ActualEntry);
2350  else
2351  // Can't use BranchProbability in general, since it takes 32-bit numbers.
2352  CSRCost = CSRCost.getFrequency() * (ActualEntry / FixedEntry);
2353 }
2354 
2355 /// \brief Collect the hint info for \p Reg.
2356 /// The results are stored into \p Out.
2357 /// \p Out is not cleared before being populated.
2358 void RAGreedy::collectHintInfo(unsigned Reg, HintsInfo &Out) {
2359  for (const MachineInstr &Instr : MRI->reg_nodbg_instructions(Reg)) {
2360  if (!Instr.isFullCopy())
2361  continue;
2362  // Look for the other end of the copy.
2363  unsigned OtherReg = Instr.getOperand(0).getReg();
2364  if (OtherReg == Reg) {
2365  OtherReg = Instr.getOperand(1).getReg();
2366  if (OtherReg == Reg)
2367  continue;
2368  }
2369  // Get the current assignment.
2370  unsigned OtherPhysReg = TargetRegisterInfo::isPhysicalRegister(OtherReg)
2371  ? OtherReg
2372  : VRM->getPhys(OtherReg);
2373  // Push the collected information.
2374  Out.push_back(HintInfo(MBFI->getBlockFreq(Instr.getParent()), OtherReg,
2375  OtherPhysReg));
2376  }
2377 }
2378 
2379 /// \brief Using the given \p List, compute the cost of the broken hints if
2380 /// \p PhysReg was used.
2381 /// \return The cost of \p List for \p PhysReg.
2382 BlockFrequency RAGreedy::getBrokenHintFreq(const HintsInfo &List,
2383  unsigned PhysReg) {
2384  BlockFrequency Cost = 0;
2385  for (const HintInfo &Info : List) {
2386  if (Info.PhysReg != PhysReg)
2387  Cost += Info.Freq;
2388  }
2389  return Cost;
2390 }
2391 
2392 /// \brief Using the register assigned to \p VirtReg, try to recolor
2393 /// all the live ranges that are copy-related with \p VirtReg.
2394 /// The recoloring is then propagated to all the live-ranges that have
2395 /// been recolored and so on, until no more copies can be coalesced or
2396 /// it is not profitable.
2397 /// For a given live range, profitability is determined by the sum of the
2398 /// frequencies of the non-identity copies it would introduce with the old
2399 /// and new register.
2400 void RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) {
2401  // We have a broken hint, check if it is possible to fix it by
2402  // reusing PhysReg for the copy-related live-ranges. Indeed, we evicted
2403  // some register and PhysReg may be available for the other live-ranges.
2404  SmallSet<unsigned, 4> Visited;
2405  SmallVector<unsigned, 2> RecoloringCandidates;
2406  HintsInfo Info;
2407  unsigned Reg = VirtReg.reg;
2408  unsigned PhysReg = VRM->getPhys(Reg);
2409  // Start the recoloring algorithm from the input live-interval, then
2410  // it will propagate to the ones that are copy-related with it.
2411  Visited.insert(Reg);
2412  RecoloringCandidates.push_back(Reg);
2413 
2414  DEBUG(dbgs() << "Trying to reconcile hints for: " << PrintReg(Reg, TRI) << '('
2415  << PrintReg(PhysReg, TRI) << ")\n");
2416 
2417  do {
2418  Reg = RecoloringCandidates.pop_back_val();
2419 
2420  // We cannot recolor physcal register.
2421  if (TargetRegisterInfo::isPhysicalRegister(Reg))
2422  continue;
2423 
2424  assert(VRM->hasPhys(Reg) && "We have unallocated variable!!");
2425 
2426  // Get the live interval mapped with this virtual register to be able
2427  // to check for the interference with the new color.
2428  LiveInterval &LI = LIS->getInterval(Reg);
2429  unsigned CurrPhys = VRM->getPhys(Reg);
2430  // Check that the new color matches the register class constraints and
2431  // that it is free for this live range.
2432  if (CurrPhys != PhysReg && (!MRI->getRegClass(Reg)->contains(PhysReg) ||
2433  Matrix->checkInterference(LI, PhysReg)))
2434  continue;
2435 
2436  DEBUG(dbgs() << PrintReg(Reg, TRI) << '(' << PrintReg(CurrPhys, TRI)
2437  << ") is recolorable.\n");
2438 
2439  // Gather the hint info.
2440  Info.clear();
2441  collectHintInfo(Reg, Info);
2442  // Check if recoloring the live-range will increase the cost of the
2443  // non-identity copies.
2444  if (CurrPhys != PhysReg) {
2445  DEBUG(dbgs() << "Checking profitability:\n");
2446  BlockFrequency OldCopiesCost = getBrokenHintFreq(Info, CurrPhys);
2447  BlockFrequency NewCopiesCost = getBrokenHintFreq(Info, PhysReg);
2448  DEBUG(dbgs() << "Old Cost: " << OldCopiesCost.getFrequency()
2449  << "\nNew Cost: " << NewCopiesCost.getFrequency() << '\n');
2450  if (OldCopiesCost < NewCopiesCost) {
2451  DEBUG(dbgs() << "=> Not profitable.\n");
2452  continue;
2453  }
2454  // At this point, the cost is either cheaper or equal. If it is
2455  // equal, we consider this is profitable because it may expose
2456  // more recoloring opportunities.
2457  DEBUG(dbgs() << "=> Profitable.\n");
2458  // Recolor the live-range.
2459  Matrix->unassign(LI);
2460  Matrix->assign(LI, PhysReg);
2461  }
2462  // Push all copy-related live-ranges to keep reconciling the broken
2463  // hints.
2464  for (const HintInfo &HI : Info) {
2465  if (Visited.insert(HI.Reg).second)
2466  RecoloringCandidates.push_back(HI.Reg);
2467  }
2468  } while (!RecoloringCandidates.empty());
2469 }
2470 
2471 /// \brief Try to recolor broken hints.
2472 /// Broken hints may be repaired by recoloring when an evicted variable
2473 /// freed up a register for a larger live-range.
2474 /// Consider the following example:
2475 /// BB1:
2476 /// a =
2477 /// b =
2478 /// BB2:
2479 /// ...
2480 /// = b
2481 /// = a
2482 /// Let us assume b gets split:
2483 /// BB1:
2484 /// a =
2485 /// b =
2486 /// BB2:
2487 /// c = b
2488 /// ...
2489 /// d = c
2490 /// = d
2491 /// = a
2492 /// Because of how the allocation work, b, c, and d may be assigned different
2493 /// colors. Now, if a gets evicted later:
2494 /// BB1:
2495 /// a =
2496 /// st a, SpillSlot
2497 /// b =
2498 /// BB2:
2499 /// c = b
2500 /// ...
2501 /// d = c
2502 /// = d
2503 /// e = ld SpillSlot
2504 /// = e
2505 /// This is likely that we can assign the same register for b, c, and d,
2506 /// getting rid of 2 copies.
2507 void RAGreedy::tryHintsRecoloring() {
2508  for (LiveInterval *LI : SetOfBrokenHints) {
2509  assert(TargetRegisterInfo::isVirtualRegister(LI->reg) &&
2510  "Recoloring is possible only for virtual registers");
2511  // Some dead defs may be around (e.g., because of debug uses).
2512  // Ignore those.
2513  if (!VRM->hasPhys(LI->reg))
2514  continue;
2515  tryHintRecoloring(*LI);
2516  }
2517 }
2518 
2519 unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
2520  SmallVectorImpl<unsigned> &NewVRegs,
2521  SmallVirtRegSet &FixedRegisters,
2522  unsigned Depth) {
2523  unsigned CostPerUseLimit = ~0u;
2524  // First try assigning a free register.
2525  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
2526  if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) {
2527  // When NewVRegs is not empty, we may have made decisions such as evicting
2528  // a virtual register, go with the earlier decisions and use the physical
2529  // register.
2530  if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) &&
2531  NewVRegs.empty()) {
2532  unsigned CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
2533  CostPerUseLimit, NewVRegs);
2534  if (CSRReg || !NewVRegs.empty())
2535  // Return now if we decide to use a CSR or create new vregs due to
2536  // pre-splitting.
2537  return CSRReg;
2538  } else
2539  return PhysReg;
2540  }
2541 
2542  LiveRangeStage Stage = getStage(VirtReg);
2543  DEBUG(dbgs() << StageName[Stage]
2544  << " Cascade " << ExtraRegInfo[VirtReg.reg].Cascade << '\n');
2545 
2546  // Try to evict a less worthy live range, but only for ranges from the primary
2547  // queue. The RS_Split ranges already failed to do this, and they should not
2548  // get a second chance until they have been split.
2549  if (Stage != RS_Split)
2550  if (unsigned PhysReg =
2551  tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit)) {
2552  unsigned Hint = MRI->getSimpleHint(VirtReg.reg);
2553  // If VirtReg has a hint and that hint is broken record this
2554  // virtual register as a recoloring candidate for broken hint.
2555  // Indeed, since we evicted a variable in its neighborhood it is
2556  // likely we can at least partially recolor some of the
2557  // copy-related live-ranges.
2558  if (Hint && Hint != PhysReg)
2559  SetOfBrokenHints.insert(&VirtReg);
2560  return PhysReg;
2561  }
2562 
2563  assert((NewVRegs.empty() || Depth) && "Cannot append to existing NewVRegs");
2564 
2565  // The first time we see a live range, don't try to split or spill.
2566  // Wait until the second time, when all smaller ranges have been allocated.
2567  // This gives a better picture of the interference to split around.
2568  if (Stage < RS_Split) {
2569  setStage(VirtReg, RS_Split);
2570  DEBUG(dbgs() << "wait for second round\n");
2571  NewVRegs.push_back(VirtReg.reg);
2572  return 0;
2573  }
2574 
2575  if (Stage < RS_Spill) {
2576  // Try splitting VirtReg or interferences.
2577  unsigned NewVRegSizeBefore = NewVRegs.size();
2578  unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs);
2579  if (PhysReg || (NewVRegs.size() - NewVRegSizeBefore))
2580  return PhysReg;
2581  }
2582 
2583  // If we couldn't allocate a register from spilling, there is probably some
2584  // invalid inline assembly. The base class wil report it.
2585  if (Stage >= RS_Done || !VirtReg.isSpillable())
2586  return tryLastChanceRecoloring(VirtReg, Order, NewVRegs, FixedRegisters,
2587  Depth);
2588 
2589  // Finally spill VirtReg itself.
2590  if (EnableDeferredSpilling && getStage(VirtReg) < RS_Memory) {
2591  // TODO: This is experimental and in particular, we do not model
2592  // the live range splitting done by spilling correctly.
2593  // We would need a deep integration with the spiller to do the
2594  // right thing here. Anyway, that is still good for early testing.
2595  setStage(VirtReg, RS_Memory);
2596  DEBUG(dbgs() << "Do as if this register is in memory\n");
2597  NewVRegs.push_back(VirtReg.reg);
2598  } else {
2599  NamedRegionTimer T("spill", "Spiller", TimerGroupName,
2600  TimerGroupDescription, TimePassesIsEnabled);
2601  LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
2602  spiller().spill(LRE);
2603  setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
2604 
2605  if (VerifyEnabled)
2606  MF->verify(this, "After spilling");
2607  }
2608 
2609  // The live virtual register requesting allocation was spilled, so tell
2610  // the caller not to allocate anything during this round.
2611  return 0;
2612 }
2613 
2614 bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
2615  DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
2616  << "********** Function: " << mf.getName() << '\n');
2617 
2618  MF = &mf;
2619  TRI = MF->getSubtarget().getRegisterInfo();
2620  TII = MF->getSubtarget().getInstrInfo();
2621  RCI.runOnMachineFunction(mf);
2622 
2623  EnableLocalReassign = EnableLocalReassignment ||
2624  MF->getSubtarget().enableRALocalReassignment(
2625  MF->getTarget().getOptLevel());
2626 
2627  if (VerifyEnabled)
2628  MF->verify(this, "Before greedy register allocator");
2629 
2630  RegAllocBase::init(getAnalysis<VirtRegMap>(),
2631  getAnalysis<LiveIntervals>(),
2632  getAnalysis<LiveRegMatrix>());
2633  Indexes = &getAnalysis<SlotIndexes>();
2634  MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
2635  DomTree = &getAnalysis<MachineDominatorTree>();
2636  SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));
2637  Loops = &getAnalysis<MachineLoopInfo>();
2638  Bundles = &getAnalysis<EdgeBundles>();
2639  SpillPlacer = &getAnalysis<SpillPlacement>();
2640  DebugVars = &getAnalysis<LiveDebugVariables>();
2641  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2642 
2643  initializeCSRCost();
2644 
2645  calculateSpillWeightsAndHints(*LIS, mf, VRM, *Loops, *MBFI);
2646 
2647  DEBUG(LIS->dump());
2648 
2649  SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops));
2650  SE.reset(new SplitEditor(*SA, *AA, *LIS, *VRM, *DomTree, *MBFI));
2651  ExtraRegInfo.clear();
2652  ExtraRegInfo.resize(MRI->getNumVirtRegs());
2653  NextCascade = 1;
2654  IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI);
2655  GlobalCand.resize(32); // This will grow as needed.
2656  SetOfBrokenHints.clear();
2657 
2658  allocatePhysRegs();
2659  tryHintsRecoloring();
2660  postOptimization();
2661 
2662  releaseMemory();
2663  return true;
2664 }
void push_back(const T &Elt)
Definition: SmallVector.h:211
iterator begin() const
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
const unsigned reg
Definition: LiveInterval.h:656
void rewind()
Start over from the beginning.
STATISTIC(NumFunctions,"Total number of functions")
size_t i
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
SlotIndex getBoundaryIndex() const
Returns the boundary index for associated with this index.
Definition: SlotIndexes.h:241
int getInstrDistance(SlotIndex other) const
Return the scaled distance from this index to the given one, where all slots on the same instruction ...
Definition: SlotIndexes.h:213
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
Definition: SlotIndexes.h:234
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:144
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:625
ArrayRef< MCPhysReg > getOrder() const
Get the allocation order without reordered hints.
bool isSpillable() const
isSpillable - Can this interval be spilled?
Definition: LiveInterval.h:754
iterator end() const
Definition: ArrayRef.h:130
SM_Speed - Overlap intervals to minimize the expected execution frequency of the inserted copies...
Definition: SplitKit.h:272
Spiller interface.
Definition: Spiller.h:25
SlotIndex FirstDef
First non-phi valno->def, or SlotIndex().
Definition: SplitKit.h:106
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
unsigned next(unsigned Limit=0)
Return the next physical register in the allocation order, or 0.
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
char & RAGreedyID
Greedy register allocator.
bool empty() const
Definition: LiveInterval.h:357
iterator end() const
Callback methods for LiveRangeEdit owners.
Definition: LiveRangeEdit.h:40
Live Register Matrix
This class represents the liveness of a register, stack slot, etc.
Definition: LiveInterval.h:153
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:172
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
Hexagon Hardware Loops
const float Hysteresis
Query interferences between a single live virtual register and a live interval union.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
static cl::opt< unsigned > LastChanceRecoloringMaxDepth("lcr-max-depth", cl::Hidden, cl::desc("Last chance recoloring max depth"), cl::init(5))
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:440
This class is basically a combination of TimeRegion and Timer.
Definition: Timer.h:160
iterator end()
Definition: LiveInterval.h:206
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getSize() const
getSize - Returns the sum of sizes of all the LiveRange's.
RegAllocBase provides the register allocation driver and interface that can be extended to add intere...
Definition: RegAllocBase.h:58
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
BorderConstraint Exit
Constraint on block exit.
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:418
MachineBasicBlock * MBB
Definition: SplitKit.h:103
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
const SmallVectorImpl< LiveInterval * > & interferingVRegs() const
SlotIndexes pass.
Definition: SlotIndexes.h:323
#define T
SplitEditor - Edit machine code and LiveIntervals for live range splitting.
Definition: SplitKit.h:236
const float huge_valf
Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
Definition: MathExtras.cpp:29
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
RegisterRegAlloc class - Track the registration of register allocators.
BlockConstraint - Entry and exit constraints for a basic block.
Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubRegIdx=0)
Prints virtual and physical registers with or without a TRI instance.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
SlotIndex LastInstr
Last instr accessing current reg.
Definition: SplitKit.h:105
unsigned get(unsigned idx) const
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
TargetInstrInfo - Interface to description of machine instruction set.
static cl::opt< unsigned > CSRFirstTimeCost("regalloc-csr-first-time-cost", cl::desc("Cost for first time use of callee-saved register."), cl::init(0), cl::Hidden)
static cl::opt< SplitEditor::ComplementSpillMode > SplitSpillMode("split-spill-mode", cl::Hidden, cl::desc("Spill mode for splitting live ranges"), cl::values(clEnumValN(SplitEditor::SM_Partition,"default","Default"), clEnumValN(SplitEditor::SM_Size,"size","Optimize for size"), clEnumValN(SplitEditor::SM_Speed,"speed","Optimize for speed")), cl::init(SplitEditor::SM_Speed))
ValuesClass values(OptsTy...Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:615
SlotIndex last()
last - Return the ending index of the last interfering range in the current block.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
const uint8_t AllocationPriority
Classes with a higher priority value are assigned first by register allocators using a greedy heurist...
bool isValid() const
Returns true if this is a valid index.
Definition: SlotIndexes.h:144
unsigned const MachineRegisterInfo * MRI
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
Cursor - The primary query interface for the block interference cache.
SM_Partition(Default) - Try to create the complement interval so it doesn't overlap any other interva...
Definition: SplitKit.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
SM_Size - Overlap intervals to minimize the number of inserted COPY instructions. ...
Definition: SplitKit.h:267
INITIALIZE_PASS_BEGIN(RAGreedy,"greedy","Greedy Register Allocator", false, false) INITIALIZE_PASS_END(RAGreedy
Represent the analysis usage information of a pass.
Greedy Register Allocator
static unsigned getNumAllocatableRegsForConstraints(const MachineInstr *MI, unsigned Reg, const TargetRegisterClass *SuperRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, const RegisterClassInfo &RCI)
Get the number of allocatable registers that match the constraints of Reg on MI and that are also in ...
unsigned Number
Basic block number (from MBB::getNumber()).
BitVector & reset()
Definition: BitVector.h:260
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
void resize(typename StorageT::size_type s)
Definition: IndexedMap.h:60
static const unsigned End
iterator begin() const
Definition: ArrayRef.h:129
bool empty() const
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:392
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
BorderConstraint Entry
Constraint on block entry.
static cl::opt< bool > EnableLocalReassignment("enable-local-reassign", cl::Hidden, cl::desc("Local reassignment can yield better allocation decisions, but ""may be compile time intensive"), cl::init(false))
const T & back() const
back - Get the last element.
Definition: ArrayRef.h:150
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
static cl::opt< unsigned > LastChanceRecoloringMaxInterference("lcr-max-interf", cl::Hidden, cl::desc("Last chance recoloring maximum number of considered"" interference at a time"), cl::init(8))
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:292
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
void calculateSpillWeightsAndHints(LiveIntervals &LIS, MachineFunction &MF, VirtRegMap *VRM, const MachineLoopInfo &MLI, const MachineBlockFrequencyInfo &MBFI, VirtRegAuxInfo::NormalizingFn norm=normalizeSpillWeight)
Compute spill weights and allocation hints for all virtual register live intervals.
SlotIndex FirstInstr
First instr accessing current reg.
Definition: SplitKit.h:104
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
bool test(unsigned Idx) const
Definition: BitVector.h:323
Additional information about basic blocks where the current variable is live.
Definition: SplitKit.h:102
Promote Memory to Register
Definition: Mem2Reg.cpp:100
SplitAnalysis - Analyze a LiveInterval, looking for live range splitting opportunities.
Definition: SplitKit.h:77
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:276
bool hasInterference()
hasInterference - Return true if the current block has any interference.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static RegisterRegAlloc greedyRegAlloc("greedy","greedy register allocator", createGreedyRegisterAllocator)
unsigned getNumRegs() const
Return the number of registers in this class.
LiveSegments::iterator SegmentIter
bool LiveOut
Current reg is live out.
Definition: SplitKit.h:108
void moveToBlock(unsigned MBBNum)
moveTo - Move cursor to basic block MBBNum.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:590
MachineFunctionProperties & set(Property P)
Spiller * createInlineSpiller(MachineFunctionPass &pass, MachineFunction &mf, VirtRegMap &vrm)
Create and return a spiller that will insert spill code directly instead of deferring though VirtRegM...
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
Definition: LiveInterval.h:360
Representation of each machine instruction.
Definition: MachineInstr.h:52
Greedy Register false
ArrayRef< unsigned > regs() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
SlotIndex endIndex() const
endNumber - return the maximum point of the range of the whole, exclusive.
Definition: LiveInterval.h:367
FunctionPass * createGreedyRegisterAllocator()
Greedy register allocation pass - This pass implements a global register allocator for optimized buil...
const NodeList & List
Definition: RDFGraph.cpp:205
#define I(x, y, z)
Definition: MD5.cpp:54
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
unsigned collectInterferingVRegs(unsigned MaxInterferingRegs=UINT_MAX)
const TargetRegisterClass * getRegClassConstraintEffectForVReg(unsigned Reg, const TargetRegisterClass *CurRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ExploreBundle=false) const
Applies the constraints (def/use) implied by this MI on Reg to the given CurRC.
void verify(const MachineRegisterInfo *MRI=nullptr) const
Walks the interval and assert if any invariants fail to hold.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def...
Definition: SlotIndexes.h:247
static cl::opt< bool > EnableDeferredSpilling("enable-deferred-spilling", cl::Hidden, cl::desc("Instead of spilling a variable right away, defer the actual ""code insertion to the end of the allocation. That way the ""allocator might still find a suitable coloring for this ""variable because of other evicted variables."), cl::init(false))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static float normalizeSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr)
Normalize the spill weight of a live interval.
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:326
bool isHint() const
Return true if the last register returned from next() was a preferred register.
static cl::opt< bool > ExhaustiveSearch("exhaustive-register-search", cl::NotHidden, cl::desc("Exhaustive Search for registers bypassing the depth ""and interference cutoffs of last chance recoloring"))
#define DEBUG(X)
Definition: Debug.h:100
IRTranslator LLVM IR MI
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: IRReader.cpp:26
unsigned size() const
bool LiveIn
Current reg is live in.
Definition: SplitKit.h:107
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:76
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
SlotIndex first()
first - Return the starting index of the first interfering range in the current block.
Properties which a MachineFunction may have at a given point in time.
void resize(size_type N)
Definition: SmallVector.h:352
bool ChangesValue
True when this block changes the value of the live range.