57 #define DEBUG_TYPE "stackcoloring"
72 cl::desc(
"Do not optimize lifetime zones that "
82 cl::desc(
"Treat stack lifetimes as starting on first use, not on START marker."));
85 STATISTIC(NumMarkerSeen,
"Number of lifetime markers found.");
86 STATISTIC(StackSpaceSaved,
"Number of bytes saved due to merging slots.");
87 STATISTIC(StackSlotMerged,
"Number of stack slot merged.");
88 STATISTIC(EscapedAllocas,
"Number of allocas that escaped the lifetime region");
255 struct BlockLifetimeInfo {
268 LivenessMap BlockLiveness;
297 unsigned NumIterations;
310 void dumpIntervals()
const;
312 void dumpBV(
const char *tag,
const BitVector &BV)
const;
316 bool removeAllMarkers();
321 unsigned collectMarkers(
unsigned NumSlot);
327 void calculateLocalLiveness();
331 bool applyFirstUse(
int Slot) {
334 if (ConservativeSlots.test(Slot))
349 void calculateLiveIntervals(
unsigned NumSlots);
361 void removeInvalidSlotRanges();
376 "stack-coloring",
"Merge disjoint stack slots",
false,
false)
382 void StackColoring::getAnalysisUsage(
AnalysisUsage &AU)
const {
393 for (
unsigned I = 0,
E = BV.
size();
I !=
E; ++
I)
399 LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
400 assert(BI != BlockLiveness.end() &&
"Block not found");
401 const BlockLifetimeInfo &BlockInfo = BI->second;
403 dumpBV(
"BEGIN", BlockInfo.Begin);
404 dumpBV(
"END", BlockInfo.End);
405 dumpBV(
"LIVE_IN", BlockInfo.LiveIn);
406 dumpBV(
"LIVE_OUT", BlockInfo.LiveOut);
430 "Expected LIFETIME_START or LIFETIME_END op");
453 if (!InterestingSlots.test(Slot))
460 if (! applyFirstUse(Slot)) {
470 int Slot = MO.getIndex();
473 if (InterestingSlots.test(Slot) && applyFirstUse(Slot)) {
487 unsigned StackColoring::collectMarkers(
unsigned NumSlot)
489 unsigned MarkersFound = 0;
490 BlockBitVecMap SeenStartMap;
491 InterestingSlots.clear();
492 InterestingSlots.resize(NumSlot);
493 ConservativeSlots.clear();
494 ConservativeSlots.resize(NumSlot);
508 BetweenStartEnd.
resize(NumSlot);
510 PE = MBB->
pred_end(); PI != PE; ++PI) {
511 BlockBitVecMap::const_iterator
I = SeenStartMap.find(*PI);
512 if (I != SeenStartMap.end()) {
513 BetweenStartEnd |= I->second;
524 InterestingSlots.set(Slot);
526 BetweenStartEnd.
set(Slot);
527 NumStartLifetimes[Slot] += 1;
529 BetweenStartEnd.
reset(Slot);
530 NumEndLifetimes[Slot] += 1;
538 DEBUG(
dbgs() <<
" marker for slot #" << Slot);
542 Markers.push_back(&MI);
548 int Slot = MO.getIndex();
551 if (! BetweenStartEnd.
test(Slot)) {
552 ConservativeSlots.set(Slot);
558 SeenStart |= BetweenStartEnd;
566 for (
unsigned slot = 0; slot < NumSlot; ++slot)
567 if (NumStartLifetimes[slot] > 1 || NumEndLifetimes[slot] > 1)
568 ConservativeSlots.
set(slot);
569 DEBUG(dumpBV(
"Conservative slots", ConservativeSlots));
579 BasicBlocks[
MBB] = BasicBlockNumbering.size();
580 BasicBlockNumbering.push_back(MBB);
583 BlockLifetimeInfo &BlockInfo = BlockLiveness[
MBB];
585 BlockInfo.Begin.resize(NumSlot);
586 BlockInfo.End.resize(NumSlot);
590 bool isStart =
false;
592 if (isLifetimeStartOrEnd(MI, slots, isStart)) {
594 assert(slots.
size() == 1 &&
"unexpected: MI ends multiple slots");
596 if (BlockInfo.Begin.test(Slot)) {
597 BlockInfo.Begin.reset(Slot);
599 BlockInfo.End.set(Slot);
601 for (
auto Slot : slots) {
602 DEBUG(
dbgs() <<
"Found a use of slot #" << Slot);
603 DEBUG(
dbgs() <<
" at BB#" << MBB->getNumber() <<
" index ");
604 DEBUG(Indexes->getInstructionIndex(MI).print(
dbgs()));
605 const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
607 DEBUG(
dbgs() <<
" with allocation: "<< Allocation->getName());
610 if (BlockInfo.End.test(Slot)) {
611 BlockInfo.End.reset(Slot);
613 BlockInfo.Begin.set(Slot);
621 NumMarkerSeen += MarkersFound;
625 void StackColoring::calculateLocalLiveness()
627 unsigned NumIters = 0;
636 LivenessMap::iterator BI = BlockLiveness.find(BB);
637 assert(BI != BlockLiveness.end() &&
"Block not found");
638 BlockLifetimeInfo &BlockInfo = BI->second;
643 PE = BB->pred_end(); PI != PE; ++PI) {
644 LivenessMap::const_iterator I = BlockLiveness.find(*PI);
645 assert(I != BlockLiveness.end() &&
"Predecessor not found");
646 LocalLiveIn |= I->second.LiveOut;
657 LocalLiveOut.
reset(BlockInfo.End);
658 LocalLiveOut |= BlockInfo.Begin;
661 if (LocalLiveIn.
test(BlockInfo.LiveIn)) {
663 BlockInfo.LiveIn |= LocalLiveIn;
667 if (LocalLiveOut.
test(BlockInfo.LiveOut)) {
669 BlockInfo.LiveOut |= LocalLiveOut;
674 NumIterations = NumIters;
677 void StackColoring::calculateLiveIntervals(
unsigned NumSlots) {
687 Finishes.
resize(NumSlots);
693 bool IsStart =
false;
694 if (!isLifetimeStartOrEnd(MI, slots, IsStart))
696 SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
697 for (
auto Slot : slots) {
699 if (!Starts[Slot].
isValid() || Starts[Slot] > ThisIndex)
700 Starts[Slot] = ThisIndex;
702 if (!Finishes[Slot].
isValid() || Finishes[Slot] < ThisIndex)
703 Finishes[Slot] = ThisIndex;
709 BlockLifetimeInfo &MBBLiveness = BlockLiveness[&
MBB];
710 for (
int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
711 pos = MBBLiveness.LiveIn.find_next(pos)) {
712 Starts[pos] = Indexes->getMBBStartIdx(&MBB);
714 for (
int pos = MBBLiveness.LiveOut.find_first(); pos != -1;
715 pos = MBBLiveness.LiveOut.find_next(pos)) {
716 Finishes[pos] = Indexes->getMBBEndIdx(&MBB);
719 for (
unsigned i = 0;
i < NumSlots; ++
i) {
741 Finishes[
i] = Indexes->getMBBEndIdx(&MBB);
746 assert(Starts[
i] && Finishes[
i] &&
"Invalid interval");
752 Intervals[
i]->addSegment(LiveInterval::Segment(S, F, ValNum));
756 SlotIndex NewStart = Indexes->getMBBStartIdx(&MBB);
757 SlotIndex NewFin = Indexes->getMBBEndIdx(&MBB);
758 Intervals[
i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
759 Intervals[
i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
765 bool StackColoring::removeAllMarkers() {
773 DEBUG(
dbgs()<<
"Removed "<<Count<<
" markers.\n");
778 unsigned FixedInstr = 0;
779 unsigned FixedMemOp = 0;
780 unsigned FixedDbg = 0;
783 for (
auto &
VI : MF->getVariableDbgInfo()) {
786 if (SlotRemap.
count(
VI.Slot)) {
787 DEBUG(
dbgs() <<
"Remapping debug info for ["
788 << cast<DILocalVariable>(
VI.Var)->getName() <<
"].\n");
789 VI.Slot = SlotRemap[
VI.Slot];
796 for (
const std::pair<int, int> &SI : SlotRemap) {
797 const AllocaInst *From = MFI->getObjectAllocation(
SI.first);
798 const AllocaInst *To = MFI->getObjectAllocation(
SI.second);
799 assert(To && From &&
"Invalid allocation object");
817 SP->adjustForColoring(From, To);
824 for (
auto &
Use : FromAI->
uses()) {
826 if (BCI->isUsedByMetadata())
855 const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
859 if (!Allocas.
count(AI))
862 MMO->setValue(Allocas[AI]);
870 int FromSlot = MO.getIndex();
877 if (!SlotRemap.count(FromSlot))
888 bool TouchesMemory = I.mayLoad() || I.mayStore();
892 SlotIndex Index = Indexes->getInstructionIndex(I);
895 "Found instruction usage outside of live range.");
900 int ToSlot = SlotRemap[FromSlot];
914 DEBUG(
dbgs()<<
"Fixed "<<FixedMemOp<<
" machine memory operands.\n");
915 DEBUG(
dbgs()<<
"Fixed "<<FixedDbg<<
" debug locations.\n");
916 DEBUG(
dbgs()<<
"Fixed "<<FixedInstr<<
" machine instructions.\n");
919 void StackColoring::removeInvalidSlotRanges() {
932 if (!I.mayLoad() && !I.mayStore())
940 int Slot = MO.getIndex();
951 SlotIndex Index = Indexes->getInstructionIndex(I);
952 if (Interval->
find(Index) == Interval->
end()) {
954 DEBUG(
dbgs()<<
"Invalidating range #"<<Slot<<
"\n");
964 for (
unsigned i=0;
i < NumSlots; ++
i) {
969 while (SlotRemap.
count(Target)) {
970 Target = SlotRemap[
Target];
978 DEBUG(
dbgs() <<
"********** Stack Coloring **********\n"
979 <<
"********** Function: "
982 MFI = &MF->getFrameInfo();
983 Indexes = &getAnalysis<SlotIndexes>();
984 SP = &getAnalysis<StackProtector>();
985 BlockLiveness.clear();
987 BasicBlockNumbering.clear();
990 VNInfoAllocator.Reset();
992 unsigned NumSlots = MFI->getObjectIndexEnd();
1002 unsigned NumMarkers = collectMarkers(NumSlots);
1004 unsigned TotalSize = 0;
1005 DEBUG(
dbgs()<<
"Found "<<NumMarkers<<
" markers and "<<NumSlots<<
" slots\n");
1008 for (
int i=0;
i < MFI->getObjectIndexEnd(); ++
i) {
1009 DEBUG(
dbgs()<<
"Slot #"<<
i<<
" - "<<MFI->getObjectSize(
i)<<
" bytes.\n");
1010 TotalSize += MFI->getObjectSize(
i);
1013 DEBUG(
dbgs()<<
"Total Stack size: "<<TotalSize<<
" bytes\n\n");
1019 DEBUG(
dbgs()<<
"Will not try to merge slots.\n");
1020 return removeAllMarkers();
1023 for (
unsigned i=0;
i < NumSlots; ++
i) {
1025 LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
1031 calculateLocalLiveness();
1032 DEBUG(
dbgs() <<
"Dataflow iterations: " << NumIterations <<
"\n");
1036 calculateLiveIntervals(NumSlots);
1037 DEBUG(dumpIntervals());
1042 removeInvalidSlotRanges();
1046 unsigned RemovedSlots = 0;
1047 unsigned ReducedSize = 0;
1050 for (
unsigned I = 0; I < NumSlots; ++
I) {
1052 SortedSlots[
I] = -1;
1063 std::stable_sort(SortedSlots.
begin(), SortedSlots.
end(),
1064 [
this](
int LHS,
int RHS) {
1066 if (LHS == -1)
return false;
1067 if (RHS == -1)
return true;
1069 return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
1072 bool Changed =
true;
1075 for (
unsigned I = 0; I < NumSlots; ++
I) {
1076 if (SortedSlots[I] == -1)
1079 for (
unsigned J=I+1; J < NumSlots; ++J) {
1080 if (SortedSlots[J] == -1)
1083 int FirstSlot = SortedSlots[
I];
1084 int SecondSlot = SortedSlots[J];
1093 SlotRemap[SecondSlot] = FirstSlot;
1094 SortedSlots[J] = -1;
1095 DEBUG(
dbgs()<<
"Merging #"<<FirstSlot<<
" and slots #"<<
1096 SecondSlot<<
" together.\n");
1097 unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
1098 MFI->getObjectAlignment(SecondSlot));
1100 assert(MFI->getObjectSize(FirstSlot) >=
1101 MFI->getObjectSize(SecondSlot) &&
1102 "Merging a small object into a larger one");
1105 ReducedSize += MFI->getObjectSize(SecondSlot);
1106 MFI->setObjectAlignment(FirstSlot, MaxAlignment);
1107 MFI->RemoveStackObject(SecondSlot);
1114 StackSpaceSaved += ReducedSize;
1115 StackSlotMerged += RemovedSlots;
1116 DEBUG(
dbgs()<<
"Merge "<<RemovedSlots<<
" slots. Saved "<<
1117 ReducedSize<<
" bytes\n");
1121 expungeSlotMap(SlotRemap, NumSlots);
1122 remapInstructions(SlotRemap);
1124 return removeAllMarkers();
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void push_back(const T &Elt)
iterator_range< use_iterator > uses()
SmallVector< WinEHHandlerType, 1 > HandlerArray
size_type size() const
size - Returns the number of bits in this bitvector.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static int getStartOrEndSlot(const MachineInstr &MI)
STATISTIC(NumFunctions,"Total number of functions")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
Interval Class - An Interval is a set of nodes defined such that every node in the interval has all o...
LiveInterval - This class represents the liveness of a register, or stack slot.
static void dump(StringRef Title, SpillInfo const &Spills)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
stack Merge disjoint stack false
void reserve(size_type N)
VNInfo - Value Number Information.
iterator_range< mop_iterator > operands()
StringRef getName() const
Return a constant reference to the value's name.
#define INITIALIZE_PASS_DEPENDENCY(depName)
A description of a memory reference used in the backend.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
char & StackColoringID
StackSlotColoring - This pass performs stack coloring and merging.
A Use represents the edge between a Value definition and its users.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
static StringRef getName(Value *V)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
void MergeSegmentsInAsValue(const LiveRange &RHS, VNInfo *LHSValNo)
Merge all of the live segments of a specific val# in RHS into this live range as the specified value ...
static cl::opt< bool > ProtectFromEscapedAllocas("protect-from-escaped-allocas", cl::init(false), cl::Hidden, cl::desc("Do not optimize lifetime zones that ""are broken"))
The user may write code that uses allocas outside of the declared lifetime zone.
This class represents a no-op cast from one type to another.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
This corresponds to the llvm.lifetime.
bool isDebugValue() const
Local Stack Slot Allocation
initializer< Ty > init(const Ty &Val)
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Allocate memory in an ever growing pool, as if by bump-pointer.
PointerType * getType() const
Overload to return most specific pointer type.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
const MachineOperand & getOperand(unsigned i) const
bool isUsedByMetadata() const
Return true if there is metadata referencing this value.
bool overlaps(const LiveRange &other) const
overlaps - Return true if the intersection of the two live ranges is not empty.
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
pred_iterator pred_begin()
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
static void Merge(const std::string &Input, const std::vector< std::string > Result, size_t NumNewFeatures)
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
bool test(unsigned Idx) const
static const char * Target
void initializeStackColoringPass(PassRegistry &)
INITIALIZE_PASS(HexagonGenMux,"hexagon-mux","Hexagon generate mux instructions", false, false) void HexagonGenMux I isValid()
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
StringRef getName() const
Return the name of the corresponding LLVM basic block, or "(null)".
Target - Wrapper for Target specific information.
VNInfo * getValNumInfo(unsigned ValNo)
getValNumInfo - Returns pointer to the specified val#.
Representation of each machine instruction.
INITIALIZE_PASS_BEGIN(StackColoring,"stack-coloring","Merge disjoint stack slots", false, false) INITIALIZE_PASS_END(StackColoring
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
static cl::opt< bool > LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use", cl::init(true), cl::Hidden, cl::desc("Treat stack lifetimes as starting on first use, not on START marker."))
Enable enhanced dataflow scheme for lifetime analysis (treat first use of stack slot as start of slot...
union llvm::WinEHHandlerType::@61 CatchObj
The CatchObj starts out life as an LLVM alloca and is eventually turned frame index.
stack Merge disjoint stack slots
iterator_range< df_iterator< T > > depth_first(const T &G)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > DisableColoring("no-stack-coloring", cl::init(false), cl::Hidden, cl::desc("Disable stack coloring"))
LLVM Value Representation.
SlotIndex - An opaque wrapper around machine indexes.
an instruction to allocate memory on the stack