59 #define DEBUG_TYPE "stackcoloring"
73 cl::desc(
"Do not optimize lifetime zones that "
76 STATISTIC(NumMarkerSeen,
"Number of lifetime markers found.");
77 STATISTIC(StackSpaceSaved,
"Number of bytes saved due to merging slots.");
78 STATISTIC(StackSlotMerged,
"Number of stack slot merged.");
79 STATISTIC(EscapedAllocas,
"Number of allocas that escaped the lifetime region");
95 struct BlockLifetimeInfo {
108 LivenessMap BlockLiveness;
142 bool removeAllMarkers();
147 unsigned collectMarkers(
unsigned NumSlot);
153 void calculateLocalLiveness();
156 void calculateLiveIntervals(
unsigned NumSlots);
168 void removeInvalidSlotRanges();
180 "stack-coloring",
"Merge disjoint stack slots",
false,
false)
187 void StackColoring::getAnalysisUsage(
AnalysisUsage &AU)
const {
197 DEBUG(
dbgs() <<
"Inspecting block #" << BasicBlocks.lookup(MBB) <<
" ["
198 << MBB->getName() <<
"]\n");
200 LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
201 assert(BI != BlockLiveness.end() &&
"Block not found");
202 const BlockLifetimeInfo &BlockInfo = BI->second;
205 for (
unsigned i=0; i < BlockInfo.Begin.size(); ++i)
206 DEBUG(
dbgs()<<BlockInfo.Begin.test(i)<<
" ");
210 for (
unsigned i=0; i < BlockInfo.End.size(); ++i)
211 DEBUG(
dbgs()<<BlockInfo.End.test(i)<<
" ");
216 for (
unsigned i=0; i < BlockInfo.LiveIn.size(); ++i)
217 DEBUG(
dbgs()<<BlockInfo.LiveIn.test(i)<<
" ");
221 for (
unsigned i=0; i < BlockInfo.LiveOut.size(); ++i)
222 DEBUG(
dbgs()<<BlockInfo.LiveOut.test(i)<<
" ");
227 unsigned StackColoring::collectMarkers(
unsigned NumSlot) {
228 unsigned MarkersFound = 0;
236 BasicBlocks[MBB] = BasicBlockNumbering.size();
237 BasicBlockNumbering.push_back(MBB);
240 BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
242 BlockInfo.Begin.resize(NumSlot);
243 BlockInfo.End.resize(NumSlot);
250 Markers.push_back(&
MI);
260 DEBUG(
dbgs()<<
"Found a lifetime marker for slot #"<<Slot<<
261 " with allocation: "<< Allocation->
getName()<<
"\n");
265 BlockInfo.Begin.set(Slot);
267 if (BlockInfo.Begin.test(Slot)) {
271 BlockInfo.Begin.reset(Slot);
273 BlockInfo.End.set(Slot);
280 NumMarkerSeen += MarkersFound;
284 void StackColoring::calculateLocalLiveness() {
291 BasicBlockNumbering.end());
292 unsigned NumSSMIters = 0;
301 if (!BBSet.count(BB))
continue;
304 LivenessMap::iterator BI = BlockLiveness.find(BB);
305 assert(BI != BlockLiveness.end() &&
"Block not found");
306 BlockLifetimeInfo &BlockInfo = BI->second;
313 PE = BB->pred_end(); PI != PE; ++PI) {
314 LivenessMap::const_iterator
I = BlockLiveness.find(*PI);
315 assert(I != BlockLiveness.end() &&
"Predecessor not found");
316 LocalLiveIn |= I->second.LiveOut;
318 LocalLiveIn |= BlockInfo.End;
319 LocalLiveIn.
reset(BlockInfo.Begin);
323 SE = BB->succ_end();
SI != SE; ++
SI) {
324 LivenessMap::const_iterator I = BlockLiveness.find(*SI);
325 assert(I != BlockLiveness.end() &&
"Successor not found");
326 LocalLiveOut |= I->second.LiveIn;
328 LocalLiveOut |= BlockInfo.Begin;
329 LocalLiveOut.
reset(BlockInfo.End);
331 LocalLiveIn |= LocalLiveOut;
332 LocalLiveOut |= LocalLiveIn;
336 LocalLiveOut.
reset(BlockInfo.End);
337 LocalLiveIn.
reset(BlockInfo.Begin);
347 LocalEndBegin &= BlockInfo.Begin;
348 LocalLiveIn |= LocalEndBegin;
349 LocalLiveOut |= LocalEndBegin;
351 if (LocalLiveIn.
test(BlockInfo.LiveIn)) {
353 BlockInfo.LiveIn |= LocalLiveIn;
355 NextBBSet.
insert(BB->pred_begin(), BB->pred_end());
358 if (LocalLiveOut.
test(BlockInfo.LiveOut)) {
360 BlockInfo.LiveOut |= LocalLiveOut;
362 NextBBSet.
insert(BB->succ_begin(), BB->succ_end());
366 BBSet = std::move(NextBBSet);
370 void StackColoring::calculateLiveIntervals(
unsigned NumSlots) {
380 Finishes.
resize(NumSlots);
384 if (
MI->getParent() != &MBB)
389 "Invalid Lifetime marker");
394 assert(Slot >= 0 &&
"Invalid slot");
396 SlotIndex ThisIndex = Indexes->getInstructionIndex(
MI);
399 if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex)
400 Starts[Slot] = ThisIndex;
402 if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex)
403 Finishes[Slot] = ThisIndex;
408 BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
409 for (
int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
410 pos = MBBLiveness.LiveIn.find_next(pos)) {
411 Starts[pos] = Indexes->getMBBStartIdx(&MBB);
413 for (
int pos = MBBLiveness.LiveOut.find_first(); pos != -1;
414 pos = MBBLiveness.LiveOut.find_next(pos)) {
415 Finishes[pos] = Indexes->getMBBEndIdx(&MBB);
418 for (
unsigned i = 0; i < NumSlots; ++i) {
419 assert(Starts[i].isValid() == Finishes[i].isValid() &&
"Unmatched range");
420 if (!Starts[i].isValid())
423 assert(Starts[i] && Finishes[i] &&
"Invalid interval");
429 Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum));
433 SlotIndex NewStart = Indexes->getMBBStartIdx(&MBB);
434 SlotIndex NewFin = Indexes->getMBBEndIdx(&MBB);
435 Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
436 Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
442 bool StackColoring::removeAllMarkers() {
445 MI->eraseFromParent();
450 DEBUG(
dbgs()<<
"Removed "<<Count<<
" markers.\n");
455 unsigned FixedInstr = 0;
456 unsigned FixedMemOp = 0;
457 unsigned FixedDbg = 0;
464 if (SlotRemap.
count(
VI.Slot)) {
465 DEBUG(
dbgs() <<
"Remapping debug info for ["
466 << cast<DILocalVariable>(
VI.Var)->getName() <<
"].\n");
467 VI.Slot = SlotRemap[
VI.Slot];
474 for (
const std::pair<int, int> &SI : SlotRemap) {
475 const AllocaInst *From = MFI->getObjectAllocation(
SI.first);
476 const AllocaInst *To = MFI->getObjectAllocation(
SI.second);
477 assert(To && From &&
"Invalid allocation object");
495 SP->adjustForColoring(From, To);
500 const_cast<AllocaInst *
>(From)->replaceAllUsesWith(Inst);
522 const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
526 if (!Allocas.
count(AI))
529 MMO->setValue(Allocas[AI]);
544 if (!SlotRemap.count(FromSlot))
555 bool TouchesMemory = I.mayLoad() || I.mayStore();
559 SlotIndex Index = Indexes->getInstructionIndex(&I);
561 assert(Interval->
find(Index) != Interval->
end() &&
562 "Found instruction usage outside of live range.");
567 int ToSlot = SlotRemap[FromSlot];
573 DEBUG(
dbgs()<<
"Fixed "<<FixedMemOp<<
" machine memory operands.\n");
574 DEBUG(
dbgs()<<
"Fixed "<<FixedDbg<<
" debug locations.\n");
575 DEBUG(
dbgs()<<
"Fixed "<<FixedInstr<<
" machine instructions.\n");
578 void StackColoring::removeInvalidSlotRanges() {
591 if (!I.mayLoad() && !I.mayStore())
610 SlotIndex Index = Indexes->getInstructionIndex(&I);
611 if (Interval->
find(Index) == Interval->
end()) {
613 DEBUG(
dbgs()<<
"Invalidating range #"<<Slot<<
"\n");
623 for (
unsigned i=0; i < NumSlots; ++i) {
625 if (SlotRemap.
count(i)) {
626 int Target = SlotRemap[i];
628 while (SlotRemap.
count(Target)) {
629 Target = SlotRemap[Target];
630 SlotRemap[i] = Target;
640 DEBUG(
dbgs() <<
"********** Stack Coloring **********\n"
641 <<
"********** Function: "
644 MFI = MF->getFrameInfo();
645 Indexes = &getAnalysis<SlotIndexes>();
646 SP = &getAnalysis<StackProtector>();
647 BlockLiveness.clear();
649 BasicBlockNumbering.clear();
652 VNInfoAllocator.Reset();
654 unsigned NumSlots = MFI->getObjectIndexEnd();
665 unsigned NumMarkers = collectMarkers(NumSlots);
667 unsigned TotalSize = 0;
668 DEBUG(
dbgs()<<
"Found "<<NumMarkers<<
" markers and "<<NumSlots<<
" slots\n");
671 for (
int i=0; i < MFI->getObjectIndexEnd(); ++i) {
672 DEBUG(
dbgs()<<
"Slot #"<<i<<
" - "<<MFI->getObjectSize(i)<<
" bytes.\n");
673 TotalSize += MFI->getObjectSize(i);
676 DEBUG(
dbgs()<<
"Total Stack size: "<<TotalSize<<
" bytes\n\n");
681 DEBUG(
dbgs()<<
"Will not try to merge slots.\n");
682 return removeAllMarkers();
685 for (
unsigned i=0; i < NumSlots; ++i) {
686 std::unique_ptr<LiveInterval> LI(
new LiveInterval(i, 0));
687 LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
693 calculateLocalLiveness();
696 calculateLiveIntervals(NumSlots);
701 removeInvalidSlotRanges();
705 unsigned RemovedSlots = 0;
706 unsigned ReducedSize = 0;
709 for (
unsigned I = 0; I < NumSlots; ++
I) {
722 std::stable_sort(SortedSlots.
begin(), SortedSlots.
end(),
723 [
this](
int LHS,
int RHS) {
725 if (LHS == -1)
return false;
726 if (RHS == -1)
return true;
728 return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
734 for (
unsigned I = 0; I < NumSlots; ++
I) {
735 if (SortedSlots[I] == -1)
738 for (
unsigned J=I+1; J < NumSlots; ++J) {
739 if (SortedSlots[J] == -1)
742 int FirstSlot = SortedSlots[
I];
743 int SecondSlot = SortedSlots[J];
746 assert (!First->
empty() && !Second->
empty() &&
"Found an empty range");
752 SlotRemap[SecondSlot] = FirstSlot;
754 DEBUG(
dbgs()<<
"Merging #"<<FirstSlot<<
" and slots #"<<
755 SecondSlot<<
" together.\n");
756 unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
757 MFI->getObjectAlignment(SecondSlot));
759 assert(MFI->getObjectSize(FirstSlot) >=
760 MFI->getObjectSize(SecondSlot) &&
761 "Merging a small object into a larger one");
764 ReducedSize += MFI->getObjectSize(SecondSlot);
765 MFI->setObjectAlignment(FirstSlot, MaxAlignment);
766 MFI->RemoveStackObject(SecondSlot);
773 StackSpaceSaved += ReducedSize;
774 StackSlotMerged += RemovedSlots;
775 DEBUG(
dbgs()<<
"Merge "<<RemovedSlots<<
" slots. Saved "<<
776 ReducedSize<<
" bytes\n");
780 expungeSlotMap(SlotRemap, NumSlots);
781 remapInstructions(SlotRemap);
783 return removeAllMarkers();
void push_back(const T &Elt)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
Interval Class - An Interval is a set of nodes defined such that every node in the interval has all o...
LiveInterval - This class represents the liveness of a register, or stack slot.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
stack Merge disjoint stack false
void reserve(size_type N)
VNInfo - Value Number Information.
StringRef getName() const
Return a constant reference to the value's name.
#define INITIALIZE_PASS_DEPENDENCY(depName)
MachineMemOperand - A description of a memory reference used in the backend.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
char & StackColoringID
StackSlotColoring - This pass performs stack coloring and merging.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
static StringRef getName(Value *V)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
void MergeSegmentsInAsValue(const LiveRange &RHS, VNInfo *LHSValNo)
Merge all of the live segments of a specific val# in RHS into this live range as the specified value ...
static cl::opt< bool > ProtectFromEscapedAllocas("protect-from-escaped-allocas", cl::init(false), cl::Hidden, cl::desc("Do not optimize lifetime zones that ""are broken"))
The user may write code that uses allocas outside of the declared lifetime zone.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
This class represents a no-op cast from one type to another.
Local Stack Slot Allocation
initializer< Ty > init(const Ty &Val)
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Allocate memory in an ever growing pool, as if by bump-pointer.
VariableDbgInfoMapTy & getVariableDbgInfo()
PointerType * getType() const
getType - Overload to return most specific pointer type
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool overlaps(const LiveRange &other) const
overlaps - Return true if the intersection of the two live ranges is not empty.
Represent the analysis usage information of a pass.
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
bool test(unsigned Idx) const
void initializeStackColoringPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Target - Wrapper for Target specific information.
VNInfo * getValNumInfo(unsigned ValNo)
getValNumInfo - Returns pointer to the specified val#.
Representation of each machine instruction.
INITIALIZE_PASS_BEGIN(StackColoring,"stack-coloring","Merge disjoint stack slots", false, false) INITIALIZE_PASS_END(StackColoring
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
stack Merge disjoint stack slots
iterator_range< df_iterator< T > > depth_first(const T &G)
std::vector< MachineBasicBlock * >::const_iterator const_succ_iterator
static cl::opt< bool > DisableColoring("no-stack-coloring", cl::init(false), cl::Hidden, cl::desc("Disable stack coloring"))
LLVM Value Representation.
SlotIndex - An opaque wrapper around machine indexes.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineModuleInfo - This class contains meta information specific to a module.
AllocaInst - an instruction to allocate memory on the stack.