LLVM  3.7.0
StackColoring.cpp
Go to the documentation of this file.
1 //===-- StackColoring.cpp -------------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass implements the stack-coloring optimization that looks for
11 // lifetime markers machine instructions (LIFESTART_BEGIN and LIFESTART_END),
12 // which represent the possible lifetime of stack slots. It attempts to
13 // merge disjoint stack slots and reduce the used stack space.
14 // NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
15 //
16 // TODO: In the future we plan to improve stack coloring in the following ways:
17 // 1. Allow merging multiple small slots into a single larger slot at different
18 // offsets.
19 // 2. Merge this pass with StackSlotColoring and allow merging of allocas with
20 // spill slots.
21 //
22 //===----------------------------------------------------------------------===//
23 
24 #include "llvm/CodeGen/Passes.h"
25 #include "llvm/ADT/BitVector.h"
28 #include "llvm/ADT/SetVector.h"
29 #include "llvm/ADT/SmallPtrSet.h"
30 #include "llvm/ADT/SparseSet.h"
31 #include "llvm/ADT/Statistic.h"
46 #include "llvm/IR/DebugInfo.h"
47 #include "llvm/IR/Dominators.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/Instructions.h"
50 #include "llvm/IR/Module.h"
52 #include "llvm/Support/Debug.h"
56 
57 using namespace llvm;
58 
59 #define DEBUG_TYPE "stackcoloring"
60 
61 static cl::opt<bool>
62 DisableColoring("no-stack-coloring",
63  cl::init(false), cl::Hidden,
64  cl::desc("Disable stack coloring"));
65 
66 /// The user may write code that uses allocas outside of the declared lifetime
67 /// zone. This can happen when the user returns a reference to a local
68 /// data-structure. We can detect these cases and decide not to optimize the
69 /// code. If this flag is enabled, we try to save the user.
70 static cl::opt<bool>
71 ProtectFromEscapedAllocas("protect-from-escaped-allocas",
72  cl::init(false), cl::Hidden,
73  cl::desc("Do not optimize lifetime zones that "
74  "are broken"));
75 
76 STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
77 STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
78 STATISTIC(StackSlotMerged, "Number of stack slot merged.");
79 STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
80 
81 //===----------------------------------------------------------------------===//
82 // StackColoring Pass
83 //===----------------------------------------------------------------------===//
84 
85 namespace {
86 /// StackColoring - A machine pass for merging disjoint stack allocations,
87 /// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
88 class StackColoring : public MachineFunctionPass {
89  MachineFrameInfo *MFI;
90  MachineFunction *MF;
91 
92  /// A class representing liveness information for a single basic block.
93  /// Each bit in the BitVector represents the liveness property
94  /// for a different stack slot.
95  struct BlockLifetimeInfo {
96  /// Which slots BEGINs in each basic block.
97  BitVector Begin;
98  /// Which slots ENDs in each basic block.
99  BitVector End;
100  /// Which slots are marked as LIVE_IN, coming into each basic block.
101  BitVector LiveIn;
102  /// Which slots are marked as LIVE_OUT, coming out of each basic block.
103  BitVector LiveOut;
104  };
105 
106  /// Maps active slots (per bit) for each basic block.
108  LivenessMap BlockLiveness;
109 
110  /// Maps serial numbers to basic blocks.
112  /// Maps basic blocks to a serial number.
113  SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering;
114 
115  /// Maps liveness intervals for each slot.
117  /// VNInfo is used for the construction of LiveIntervals.
118  VNInfo::Allocator VNInfoAllocator;
119  /// SlotIndex analysis object.
120  SlotIndexes *Indexes;
121  /// The stack protector object.
123 
124  /// The list of lifetime markers found. These markers are to be removed
125  /// once the coloring is done.
127 
128 public:
129  static char ID;
130  StackColoring() : MachineFunctionPass(ID) {
132  }
133  void getAnalysisUsage(AnalysisUsage &AU) const override;
134  bool runOnMachineFunction(MachineFunction &MF) override;
135 
136 private:
137  /// Debug.
138  void dump() const;
139 
140  /// Removes all of the lifetime marker instructions from the function.
141  /// \returns true if any markers were removed.
142  bool removeAllMarkers();
143 
144  /// Scan the machine function and find all of the lifetime markers.
145  /// Record the findings in the BEGIN and END vectors.
146  /// \returns the number of markers found.
147  unsigned collectMarkers(unsigned NumSlot);
148 
149  /// Perform the dataflow calculation and calculate the lifetime for each of
150  /// the slots, based on the BEGIN/END vectors. Set the LifetimeLIVE_IN and
151  /// LifetimeLIVE_OUT maps that represent which stack slots are live coming
152  /// in and out blocks.
153  void calculateLocalLiveness();
154 
155  /// Construct the LiveIntervals for the slots.
156  void calculateLiveIntervals(unsigned NumSlots);
157 
158  /// Go over the machine function and change instructions which use stack
159  /// slots to use the joint slots.
160  void remapInstructions(DenseMap<int, int> &SlotRemap);
161 
162  /// The input program may contain instructions which are not inside lifetime
163  /// markers. This can happen due to a bug in the compiler or due to a bug in
164  /// user code (for example, returning a reference to a local variable).
165  /// This procedure checks all of the instructions in the function and
166  /// invalidates lifetime ranges which do not contain all of the instructions
167  /// which access that frame slot.
168  void removeInvalidSlotRanges();
169 
170  /// Map entries which point to other entries to their destination.
171  /// A->B->C becomes A->C.
172  void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
173 };
174 } // end anonymous namespace
175 
176 char StackColoring::ID = 0;
178 
179 INITIALIZE_PASS_BEGIN(StackColoring,
180  "stack-coloring", "Merge disjoint stack slots", false, false)
184 INITIALIZE_PASS_END(StackColoring,
185  "stack-coloring", "Merge disjoint stack slots", false, false)
186 
187 void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
188  AU.addRequired<MachineDominatorTree>();
189  AU.addPreserved<MachineDominatorTree>();
190  AU.addRequired<SlotIndexes>();
191  AU.addRequired<StackProtector>();
193 }
194 
195 void StackColoring::dump() const {
196  for (MachineBasicBlock *MBB : depth_first(MF)) {
197  DEBUG(dbgs() << "Inspecting block #" << BasicBlocks.lookup(MBB) << " ["
198  << MBB->getName() << "]\n");
199 
200  LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
201  assert(BI != BlockLiveness.end() && "Block not found");
202  const BlockLifetimeInfo &BlockInfo = BI->second;
203 
204  DEBUG(dbgs()<<"BEGIN : {");
205  for (unsigned i=0; i < BlockInfo.Begin.size(); ++i)
206  DEBUG(dbgs()<<BlockInfo.Begin.test(i)<<" ");
207  DEBUG(dbgs()<<"}\n");
208 
209  DEBUG(dbgs()<<"END : {");
210  for (unsigned i=0; i < BlockInfo.End.size(); ++i)
211  DEBUG(dbgs()<<BlockInfo.End.test(i)<<" ");
212 
213  DEBUG(dbgs()<<"}\n");
214 
215  DEBUG(dbgs()<<"LIVE_IN: {");
216  for (unsigned i=0; i < BlockInfo.LiveIn.size(); ++i)
217  DEBUG(dbgs()<<BlockInfo.LiveIn.test(i)<<" ");
218 
219  DEBUG(dbgs()<<"}\n");
220  DEBUG(dbgs()<<"LIVEOUT: {");
221  for (unsigned i=0; i < BlockInfo.LiveOut.size(); ++i)
222  DEBUG(dbgs()<<BlockInfo.LiveOut.test(i)<<" ");
223  DEBUG(dbgs()<<"}\n");
224  }
225 }
226 
227 unsigned StackColoring::collectMarkers(unsigned NumSlot) {
228  unsigned MarkersFound = 0;
229  // Scan the function to find all lifetime markers.
230  // NOTE: We use a reverse-post-order iteration to ensure that we obtain a
231  // deterministic numbering, and because we'll need a post-order iteration
232  // later for solving the liveness dataflow problem.
233  for (MachineBasicBlock *MBB : depth_first(MF)) {
234 
235  // Assign a serial number to this basic block.
236  BasicBlocks[MBB] = BasicBlockNumbering.size();
237  BasicBlockNumbering.push_back(MBB);
238 
239  // Keep a reference to avoid repeated lookups.
240  BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
241 
242  BlockInfo.Begin.resize(NumSlot);
243  BlockInfo.End.resize(NumSlot);
244 
245  for (MachineInstr &MI : *MBB) {
246  if (MI.getOpcode() != TargetOpcode::LIFETIME_START &&
247  MI.getOpcode() != TargetOpcode::LIFETIME_END)
248  continue;
249 
250  Markers.push_back(&MI);
251 
252  bool IsStart = MI.getOpcode() == TargetOpcode::LIFETIME_START;
253  const MachineOperand &MO = MI.getOperand(0);
254  unsigned Slot = MO.getIndex();
255 
256  MarkersFound++;
257 
258  const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
259  if (Allocation) {
260  DEBUG(dbgs()<<"Found a lifetime marker for slot #"<<Slot<<
261  " with allocation: "<< Allocation->getName()<<"\n");
262  }
263 
264  if (IsStart) {
265  BlockInfo.Begin.set(Slot);
266  } else {
267  if (BlockInfo.Begin.test(Slot)) {
268  // Allocas that start and end within a single block are handled
269  // specially when computing the LiveIntervals to avoid pessimizing
270  // the liveness propagation.
271  BlockInfo.Begin.reset(Slot);
272  } else {
273  BlockInfo.End.set(Slot);
274  }
275  }
276  }
277  }
278 
279  // Update statistics.
280  NumMarkerSeen += MarkersFound;
281  return MarkersFound;
282 }
283 
284 void StackColoring::calculateLocalLiveness() {
285  // Perform a standard reverse dataflow computation to solve for
286  // global liveness. The BEGIN set here is equivalent to KILL in the standard
287  // formulation, and END is equivalent to GEN. The result of this computation
288  // is a map from blocks to bitvectors where the bitvectors represent which
289  // allocas are live in/out of that block.
290  SmallPtrSet<const MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
291  BasicBlockNumbering.end());
292  unsigned NumSSMIters = 0;
293  bool changed = true;
294  while (changed) {
295  changed = false;
296  ++NumSSMIters;
297 
299 
300  for (const MachineBasicBlock *BB : BasicBlockNumbering) {
301  if (!BBSet.count(BB)) continue;
302 
303  // Use an iterator to avoid repeated lookups.
304  LivenessMap::iterator BI = BlockLiveness.find(BB);
305  assert(BI != BlockLiveness.end() && "Block not found");
306  BlockLifetimeInfo &BlockInfo = BI->second;
307 
308  BitVector LocalLiveIn;
309  BitVector LocalLiveOut;
310 
311  // Forward propagation from begins to ends.
312  for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
313  PE = BB->pred_end(); PI != PE; ++PI) {
314  LivenessMap::const_iterator I = BlockLiveness.find(*PI);
315  assert(I != BlockLiveness.end() && "Predecessor not found");
316  LocalLiveIn |= I->second.LiveOut;
317  }
318  LocalLiveIn |= BlockInfo.End;
319  LocalLiveIn.reset(BlockInfo.Begin);
320 
321  // Reverse propagation from ends to begins.
322  for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(),
323  SE = BB->succ_end(); SI != SE; ++SI) {
324  LivenessMap::const_iterator I = BlockLiveness.find(*SI);
325  assert(I != BlockLiveness.end() && "Successor not found");
326  LocalLiveOut |= I->second.LiveIn;
327  }
328  LocalLiveOut |= BlockInfo.Begin;
329  LocalLiveOut.reset(BlockInfo.End);
330 
331  LocalLiveIn |= LocalLiveOut;
332  LocalLiveOut |= LocalLiveIn;
333 
334  // After adopting the live bits, we need to turn-off the bits which
335  // are de-activated in this block.
336  LocalLiveOut.reset(BlockInfo.End);
337  LocalLiveIn.reset(BlockInfo.Begin);
338 
339  // If we have both BEGIN and END markers in the same basic block then
340  // we know that the BEGIN marker comes after the END, because we already
341  // handle the case where the BEGIN comes before the END when collecting
342  // the markers (and building the BEGIN/END vectore).
343  // Want to enable the LIVE_IN and LIVE_OUT of slots that have both
344  // BEGIN and END because it means that the value lives before and after
345  // this basic block.
346  BitVector LocalEndBegin = BlockInfo.End;
347  LocalEndBegin &= BlockInfo.Begin;
348  LocalLiveIn |= LocalEndBegin;
349  LocalLiveOut |= LocalEndBegin;
350 
351  if (LocalLiveIn.test(BlockInfo.LiveIn)) {
352  changed = true;
353  BlockInfo.LiveIn |= LocalLiveIn;
354 
355  NextBBSet.insert(BB->pred_begin(), BB->pred_end());
356  }
357 
358  if (LocalLiveOut.test(BlockInfo.LiveOut)) {
359  changed = true;
360  BlockInfo.LiveOut |= LocalLiveOut;
361 
362  NextBBSet.insert(BB->succ_begin(), BB->succ_end());
363  }
364  }
365 
366  BBSet = std::move(NextBBSet);
367  }// while changed.
368 }
369 
370 void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
373 
374  // For each block, find which slots are active within this block
375  // and update the live intervals.
376  for (const MachineBasicBlock &MBB : *MF) {
377  Starts.clear();
378  Starts.resize(NumSlots);
379  Finishes.clear();
380  Finishes.resize(NumSlots);
381 
382  // Create the interval for the basic blocks with lifetime markers in them.
383  for (const MachineInstr *MI : Markers) {
384  if (MI->getParent() != &MBB)
385  continue;
386 
387  assert((MI->getOpcode() == TargetOpcode::LIFETIME_START ||
388  MI->getOpcode() == TargetOpcode::LIFETIME_END) &&
389  "Invalid Lifetime marker");
390 
391  bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START;
392  const MachineOperand &Mo = MI->getOperand(0);
393  int Slot = Mo.getIndex();
394  assert(Slot >= 0 && "Invalid slot");
395 
396  SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
397 
398  if (IsStart) {
399  if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex)
400  Starts[Slot] = ThisIndex;
401  } else {
402  if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex)
403  Finishes[Slot] = ThisIndex;
404  }
405  }
406 
407  // Create the interval of the blocks that we previously found to be 'alive'.
408  BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
409  for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
410  pos = MBBLiveness.LiveIn.find_next(pos)) {
411  Starts[pos] = Indexes->getMBBStartIdx(&MBB);
412  }
413  for (int pos = MBBLiveness.LiveOut.find_first(); pos != -1;
414  pos = MBBLiveness.LiveOut.find_next(pos)) {
415  Finishes[pos] = Indexes->getMBBEndIdx(&MBB);
416  }
417 
418  for (unsigned i = 0; i < NumSlots; ++i) {
419  assert(Starts[i].isValid() == Finishes[i].isValid() && "Unmatched range");
420  if (!Starts[i].isValid())
421  continue;
422 
423  assert(Starts[i] && Finishes[i] && "Invalid interval");
424  VNInfo *ValNum = Intervals[i]->getValNumInfo(0);
425  SlotIndex S = Starts[i];
426  SlotIndex F = Finishes[i];
427  if (S < F) {
428  // We have a single consecutive region.
429  Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum));
430  } else {
431  // We have two non-consecutive regions. This happens when
432  // LIFETIME_START appears after the LIFETIME_END marker.
433  SlotIndex NewStart = Indexes->getMBBStartIdx(&MBB);
434  SlotIndex NewFin = Indexes->getMBBEndIdx(&MBB);
435  Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
436  Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
437  }
438  }
439  }
440 }
441 
442 bool StackColoring::removeAllMarkers() {
443  unsigned Count = 0;
444  for (MachineInstr *MI : Markers) {
445  MI->eraseFromParent();
446  Count++;
447  }
448  Markers.clear();
449 
450  DEBUG(dbgs()<<"Removed "<<Count<<" markers.\n");
451  return Count;
452 }
453 
454 void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
455  unsigned FixedInstr = 0;
456  unsigned FixedMemOp = 0;
457  unsigned FixedDbg = 0;
458  MachineModuleInfo *MMI = &MF->getMMI();
459 
460  // Remap debug information that refers to stack slots.
461  for (auto &VI : MMI->getVariableDbgInfo()) {
462  if (!VI.Var)
463  continue;
464  if (SlotRemap.count(VI.Slot)) {
465  DEBUG(dbgs() << "Remapping debug info for ["
466  << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
467  VI.Slot = SlotRemap[VI.Slot];
468  FixedDbg++;
469  }
470  }
471 
472  // Keep a list of *allocas* which need to be remapped.
474  for (const std::pair<int, int> &SI : SlotRemap) {
475  const AllocaInst *From = MFI->getObjectAllocation(SI.first);
476  const AllocaInst *To = MFI->getObjectAllocation(SI.second);
477  assert(To && From && "Invalid allocation object");
478  Allocas[From] = To;
479 
480  // AA might be used later for instruction scheduling, and we need it to be
481  // able to deduce the correct aliasing releationships between pointers
482  // derived from the alloca being remapped and the target of that remapping.
483  // The only safe way, without directly informing AA about the remapping
484  // somehow, is to directly update the IR to reflect the change being made
485  // here.
486  Instruction *Inst = const_cast<AllocaInst *>(To);
487  if (From->getType() != To->getType()) {
488  BitCastInst *Cast = new BitCastInst(Inst, From->getType());
489  Cast->insertAfter(Inst);
490  Inst = Cast;
491  }
492 
493  // Allow the stack protector to adjust its value map to account for the
494  // upcoming replacement.
495  SP->adjustForColoring(From, To);
496 
497  // Note that this will not replace uses in MMOs (which we'll update below),
498  // or anywhere else (which is why we won't delete the original
499  // instruction).
500  const_cast<AllocaInst *>(From)->replaceAllUsesWith(Inst);
501  }
502 
503  // Remap all instructions to the new stack slots.
504  for (MachineBasicBlock &BB : *MF)
505  for (MachineInstr &I : BB) {
506  // Skip lifetime markers. We'll remove them soon.
507  if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
508  I.getOpcode() == TargetOpcode::LIFETIME_END)
509  continue;
510 
511  // Update the MachineMemOperand to use the new alloca.
512  for (MachineMemOperand *MMO : I.memoperands()) {
513  // FIXME: In order to enable the use of TBAA when using AA in CodeGen,
514  // we'll also need to update the TBAA nodes in MMOs with values
515  // derived from the merged allocas. When doing this, we'll need to use
516  // the same variant of GetUnderlyingObjects that is used by the
517  // instruction scheduler (that can look through ptrtoint/inttoptr
518  // pairs).
519 
520  // We've replaced IR-level uses of the remapped allocas, so we only
521  // need to replace direct uses here.
522  const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
523  if (!AI)
524  continue;
525 
526  if (!Allocas.count(AI))
527  continue;
528 
529  MMO->setValue(Allocas[AI]);
530  FixedMemOp++;
531  }
532 
533  // Update all of the machine instruction operands.
534  for (MachineOperand &MO : I.operands()) {
535  if (!MO.isFI())
536  continue;
537  int FromSlot = MO.getIndex();
538 
539  // Don't touch arguments.
540  if (FromSlot<0)
541  continue;
542 
543  // Only look at mapped slots.
544  if (!SlotRemap.count(FromSlot))
545  continue;
546 
547  // In a debug build, check that the instruction that we are modifying is
548  // inside the expected live range. If the instruction is not inside
549  // the calculated range then it means that the alloca usage moved
550  // outside of the lifetime markers, or that the user has a bug.
551  // NOTE: Alloca address calculations which happen outside the lifetime
552  // zone are are okay, despite the fact that we don't have a good way
553  // for validating all of the usages of the calculation.
554 #ifndef NDEBUG
555  bool TouchesMemory = I.mayLoad() || I.mayStore();
556  // If we *don't* protect the user from escaped allocas, don't bother
557  // validating the instructions.
558  if (!I.isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
559  SlotIndex Index = Indexes->getInstructionIndex(&I);
560  const LiveInterval *Interval = &*Intervals[FromSlot];
561  assert(Interval->find(Index) != Interval->end() &&
562  "Found instruction usage outside of live range.");
563  }
564 #endif
565 
566  // Fix the machine instructions.
567  int ToSlot = SlotRemap[FromSlot];
568  MO.setIndex(ToSlot);
569  FixedInstr++;
570  }
571  }
572 
573  DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
574  DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
575  DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
576 }
577 
578 void StackColoring::removeInvalidSlotRanges() {
579  for (MachineBasicBlock &BB : *MF)
580  for (MachineInstr &I : BB) {
581  if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
582  I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugValue())
583  continue;
584 
585  // Some intervals are suspicious! In some cases we find address
586  // calculations outside of the lifetime zone, but not actual memory
587  // read or write. Memory accesses outside of the lifetime zone are a clear
588  // violation, but address calculations are okay. This can happen when
589  // GEPs are hoisted outside of the lifetime zone.
590  // So, in here we only check instructions which can read or write memory.
591  if (!I.mayLoad() && !I.mayStore())
592  continue;
593 
594  // Check all of the machine operands.
595  for (const MachineOperand &MO : I.operands()) {
596  if (!MO.isFI())
597  continue;
598 
599  int Slot = MO.getIndex();
600 
601  if (Slot<0)
602  continue;
603 
604  if (Intervals[Slot]->empty())
605  continue;
606 
607  // Check that the used slot is inside the calculated lifetime range.
608  // If it is not, warn about it and invalidate the range.
609  LiveInterval *Interval = &*Intervals[Slot];
610  SlotIndex Index = Indexes->getInstructionIndex(&I);
611  if (Interval->find(Index) == Interval->end()) {
612  Interval->clear();
613  DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
614  EscapedAllocas++;
615  }
616  }
617  }
618 }
619 
620 void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
621  unsigned NumSlots) {
622  // Expunge slot remap map.
623  for (unsigned i=0; i < NumSlots; ++i) {
624  // If we are remapping i
625  if (SlotRemap.count(i)) {
626  int Target = SlotRemap[i];
627  // As long as our target is mapped to something else, follow it.
628  while (SlotRemap.count(Target)) {
629  Target = SlotRemap[Target];
630  SlotRemap[i] = Target;
631  }
632  }
633  }
634 }
635 
636 bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
637  if (skipOptnoneFunction(*Func.getFunction()))
638  return false;
639 
640  DEBUG(dbgs() << "********** Stack Coloring **********\n"
641  << "********** Function: "
642  << ((const Value*)Func.getFunction())->getName() << '\n');
643  MF = &Func;
644  MFI = MF->getFrameInfo();
645  Indexes = &getAnalysis<SlotIndexes>();
646  SP = &getAnalysis<StackProtector>();
647  BlockLiveness.clear();
648  BasicBlocks.clear();
649  BasicBlockNumbering.clear();
650  Markers.clear();
651  Intervals.clear();
652  VNInfoAllocator.Reset();
653 
654  unsigned NumSlots = MFI->getObjectIndexEnd();
655 
656  // If there are no stack slots then there are no markers to remove.
657  if (!NumSlots)
658  return false;
659 
660  SmallVector<int, 8> SortedSlots;
661 
662  SortedSlots.reserve(NumSlots);
663  Intervals.reserve(NumSlots);
664 
665  unsigned NumMarkers = collectMarkers(NumSlots);
666 
667  unsigned TotalSize = 0;
668  DEBUG(dbgs()<<"Found "<<NumMarkers<<" markers and "<<NumSlots<<" slots\n");
669  DEBUG(dbgs()<<"Slot structure:\n");
670 
671  for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
672  DEBUG(dbgs()<<"Slot #"<<i<<" - "<<MFI->getObjectSize(i)<<" bytes.\n");
673  TotalSize += MFI->getObjectSize(i);
674  }
675 
676  DEBUG(dbgs()<<"Total Stack size: "<<TotalSize<<" bytes\n\n");
677 
678  // Don't continue because there are not enough lifetime markers, or the
679  // stack is too small, or we are told not to optimize the slots.
680  if (NumMarkers < 2 || TotalSize < 16 || DisableColoring) {
681  DEBUG(dbgs()<<"Will not try to merge slots.\n");
682  return removeAllMarkers();
683  }
684 
685  for (unsigned i=0; i < NumSlots; ++i) {
686  std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
687  LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
688  Intervals.push_back(std::move(LI));
689  SortedSlots.push_back(i);
690  }
691 
692  // Calculate the liveness of each block.
693  calculateLocalLiveness();
694 
695  // Propagate the liveness information.
696  calculateLiveIntervals(NumSlots);
697 
698  // Search for allocas which are used outside of the declared lifetime
699  // markers.
701  removeInvalidSlotRanges();
702 
703  // Maps old slots to new slots.
704  DenseMap<int, int> SlotRemap;
705  unsigned RemovedSlots = 0;
706  unsigned ReducedSize = 0;
707 
708  // Do not bother looking at empty intervals.
709  for (unsigned I = 0; I < NumSlots; ++I) {
710  if (Intervals[SortedSlots[I]]->empty())
711  SortedSlots[I] = -1;
712  }
713 
714  // This is a simple greedy algorithm for merging allocas. First, sort the
715  // slots, placing the largest slots first. Next, perform an n^2 scan and look
716  // for disjoint slots. When you find disjoint slots, merge the samller one
717  // into the bigger one and update the live interval. Remove the small alloca
718  // and continue.
719 
720  // Sort the slots according to their size. Place unused slots at the end.
721  // Use stable sort to guarantee deterministic code generation.
722  std::stable_sort(SortedSlots.begin(), SortedSlots.end(),
723  [this](int LHS, int RHS) {
724  // We use -1 to denote a uninteresting slot. Place these slots at the end.
725  if (LHS == -1) return false;
726  if (RHS == -1) return true;
727  // Sort according to size.
728  return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
729  });
730 
731  bool Changed = true;
732  while (Changed) {
733  Changed = false;
734  for (unsigned I = 0; I < NumSlots; ++I) {
735  if (SortedSlots[I] == -1)
736  continue;
737 
738  for (unsigned J=I+1; J < NumSlots; ++J) {
739  if (SortedSlots[J] == -1)
740  continue;
741 
742  int FirstSlot = SortedSlots[I];
743  int SecondSlot = SortedSlots[J];
744  LiveInterval *First = &*Intervals[FirstSlot];
745  LiveInterval *Second = &*Intervals[SecondSlot];
746  assert (!First->empty() && !Second->empty() && "Found an empty range");
747 
748  // Merge disjoint slots.
749  if (!First->overlaps(*Second)) {
750  Changed = true;
751  First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
752  SlotRemap[SecondSlot] = FirstSlot;
753  SortedSlots[J] = -1;
754  DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
755  SecondSlot<<" together.\n");
756  unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
757  MFI->getObjectAlignment(SecondSlot));
758 
759  assert(MFI->getObjectSize(FirstSlot) >=
760  MFI->getObjectSize(SecondSlot) &&
761  "Merging a small object into a larger one");
762 
763  RemovedSlots+=1;
764  ReducedSize += MFI->getObjectSize(SecondSlot);
765  MFI->setObjectAlignment(FirstSlot, MaxAlignment);
766  MFI->RemoveStackObject(SecondSlot);
767  }
768  }
769  }
770  }// While changed.
771 
772  // Record statistics.
773  StackSpaceSaved += ReducedSize;
774  StackSlotMerged += RemovedSlots;
775  DEBUG(dbgs()<<"Merge "<<RemovedSlots<<" slots. Saved "<<
776  ReducedSize<<" bytes\n");
777 
778  // Scan the entire function and update all machine operands that use frame
779  // indices to use the remapped frame index.
780  expungeSlotMap(SlotRemap, NumSlots);
781  remapInstructions(SlotRemap);
782 
783  return removeAllMarkers();
784 }
stack coloring
void push_back(const T &Elt)
Definition: SmallVector.h:222
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
Interval Class - An Interval is a set of nodes defined such that every node in the interval has all o...
Definition: Interval.h:37
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:588
F(f)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
stack Merge disjoint stack false
void reserve(size_type N)
Definition: SmallVector.h:401
VNInfo - Value Number Information.
Definition: LiveInterval.h:45
bool empty() const
Definition: LiveInterval.h:353
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
MachineMemOperand - A description of a memory reference used in the backend.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
char & StackColoringID
StackSlotColoring - This pass performs stack coloring and merging.
iterator end()
Definition: LiveInterval.h:206
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
static StringRef getName(Value *V)
void setIndex(int Idx)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
void MergeSegmentsInAsValue(const LiveRange &RHS, VNInfo *LHSValNo)
Merge all of the live segments of a specific val# in RHS into this live range as the specified value ...
static cl::opt< bool > ProtectFromEscapedAllocas("protect-from-escaped-allocas", cl::init(false), cl::Hidden, cl::desc("Do not optimize lifetime zones that ""are broken"))
The user may write code that uses allocas outside of the declared lifetime zone.
SlotIndexes pass.
Definition: SlotIndexes.h:334
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
This class represents a no-op cast from one type to another.
Local Stack Slot Allocation
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:135
VariableDbgInfoMapTy & getVariableDbgInfo()
PointerType * getType() const
getType - Overload to return most specific pointer type
Definition: Instructions.h:115
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
bool overlaps(const LiveRange &other) const
overlaps - Return true if the intersection of the two live ranges is not empty.
Definition: LiveInterval.h:419
Represent the analysis usage information of a pass.
BitVector & reset()
Definition: BitVector.h:259
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Module.h This file contains the declarations for the Module class.
bool test(unsigned Idx) const
Definition: BitVector.h:322
SI Fix CF Live Intervals
void initializeStackColoringPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:119
Target - Wrapper for Target specific information.
VNInfo * getValNumInfo(unsigned ValNo)
getValNumInfo - Returns pointer to the specified val#.
Definition: LiveInterval.h:292
Representation of each machine instruction.
Definition: MachineInstr.h:51
INITIALIZE_PASS_BEGIN(StackColoring,"stack-coloring","Merge disjoint stack slots", false, false) INITIALIZE_PASS_END(StackColoring
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
Definition: Instruction.cpp:82
#define I(x, y, z)
Definition: MD5.cpp:54
safe stack
Definition: SafeStack.cpp:606
stack Merge disjoint stack slots
iterator_range< df_iterator< T > > depth_first(const T &G)
std::vector< MachineBasicBlock * >::const_iterator const_succ_iterator
static cl::opt< bool > DisableColoring("no-stack-coloring", cl::init(false), cl::Hidden, cl::desc("Disable stack coloring"))
LLVM Value Representation.
Definition: Value.h:69
#define DEBUG(X)
Definition: Debug.h:92
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:92
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineModuleInfo - This class contains meta information specific to a module.
AllocaInst - an instruction to allocate memory on the stack.
Definition: Instructions.h:76
void resize(size_type N)
Definition: SmallVector.h:376