62 #define DEBUG_TYPE "regalloc"
70 cl::desc(
"Attempt coalescing during PBQP register allocation."),
76 cl::desc(
"Dump graphs for each function/round in the compilation unit."),
92 RegAllocPBQP(
char *cPassID =
nullptr)
101 const char* getPassName()
const override {
102 return "PBQP Register Allocator";
113 typedef std::map<const LiveInterval*, unsigned> LI2NodeMap;
114 typedef std::vector<const LiveInterval*> Node2LIMap;
115 typedef std::vector<unsigned> AllowedSet;
116 typedef std::vector<AllowedSet> AllowedSetMap;
117 typedef std::pair<unsigned, unsigned> RegPair;
118 typedef std::map<RegPair, PBQP::PBQPNum> CoalesceMap;
119 typedef std::set<unsigned> RegSet;
123 RegSet VRegsToAlloc, EmptyIntervalVRegs;
165 if (SpillCost == 0.0)
168 SpillCost += MinSpillCost;
181 typedef std::pair<AllowedRegVecPtr, AllowedRegVecPtr> IKey;
184 typedef std::pair<PBQP::GraphBase::NodeId, PBQP::GraphBase::NodeId> IEdgeKey;
189 const DisjointAllowedRegsCache &D)
const {
197 return D.count(IKey(NRegs, MRegs)) > 0;
199 return D.count(IKey(MRegs, NRegs)) > 0;
204 DisjointAllowedRegsCache &D) {
208 assert(NRegs != MRegs &&
"AllowedRegs can not be disjoint with itself");
211 D.insert(IKey(NRegs, MRegs));
213 D.insert(IKey(MRegs, NRegs));
220 typedef std::tuple<LiveInterval*, size_t, PBQP::GraphBase::NodeId>
223 static SlotIndex getStartPoint(
const IntervalInfo &
I) {
224 return std::get<0>(
I)->segments[std::get<1>(I)].start;
227 static SlotIndex getEndPoint(
const IntervalInfo &
I) {
228 return std::get<0>(
I)->segments[std::get<1>(I)].end;
232 return std::get<2>(
I);
235 static bool lowestStartPoint(
const IntervalInfo &I1,
236 const IntervalInfo &I2) {
239 return getStartPoint(I1) > getStartPoint(I2);
242 static bool lowestEndPoint(
const IntervalInfo &I1,
243 const IntervalInfo &I2) {
256 return std::get<0>(I1)->reg < std::get<0>(I2)->reg;
259 static bool isAtLastSegment(
const IntervalInfo &
I) {
260 return std::get<1>(
I) == std::get<0>(I)->size() - 1;
263 static IntervalInfo nextSegment(
const IntervalInfo &I) {
264 return std::make_tuple(std::get<0>(I), std::get<1>(I) + 1, std::get<2>(I));
287 DisjointAllowedRegsCache D;
289 typedef std::set<IntervalInfo, decltype(&lowestEndPoint)> IntervalSet;
290 typedef std::priority_queue<IntervalInfo, std::vector<IntervalInfo>,
291 decltype(&lowestStartPoint)> IntervalQueue;
292 IntervalSet Active(lowestEndPoint);
293 IntervalQueue Inactive(lowestStartPoint);
299 assert(!LI.
empty() &&
"PBQP graph contains node for empty interval");
300 Inactive.push(std::make_tuple(&LI, 0, NId));
303 while (!Inactive.empty()) {
306 IntervalInfo Cur = Inactive.top();
309 IntervalSet::iterator RetireItr = Active.
begin();
310 while (RetireItr != Active.end() &&
311 (getEndPoint(*RetireItr) <= getStartPoint(Cur))) {
314 if (!isAtLastSegment(*RetireItr))
315 Inactive.push(nextSegment(*RetireItr));
319 Active.erase(Active.begin(), RetireItr);
323 Cur = Inactive.top();
329 for (
const auto &
A : Active) {
334 if (haveDisjointAllowedRegs(G, NId, MId, D))
338 IEdgeKey EK(
std::min(NId, MId), std::max(NId, MId));
343 if (!createInterferenceEdge(G, NId, MId, C))
344 setDisjointAllowedRegs(G, NId, MId, D);
366 *G.
getMetadata().MF.getSubtarget().getRegisterInfo();
371 IKey K(&NRegs, &MRegs);
372 IMatrixCache::iterator I = C.find(K);
379 bool NodesInterfere =
false;
380 for (
unsigned I = 0; I != NRegs.size(); ++
I) {
381 unsigned PRegN = NRegs[
I];
382 for (
unsigned J = 0; J != MRegs.size(); ++J) {
383 unsigned PRegM = MRegs[J];
385 M[I + 1][J + 1] = std::numeric_limits<PBQP::PBQPNum>::infinity();
386 NodesInterfere =
true;
411 for (
const auto &MBB : MF) {
412 for (
const auto &
MI : MBB) {
415 if (!
CP.setRegisters(&
MI) ||
CP.getSrcReg() ==
CP.getDstReg())
418 unsigned DstReg =
CP.getDstReg();
419 unsigned SrcReg =
CP.getSrcReg();
425 if (!MF.getRegInfo().isAllocatable(DstReg))
430 const PBQPRAGraph::NodeMetadata::AllowedRegVector &
Allowed =
433 unsigned PRegOpt = 0;
434 while (PRegOpt < Allowed.size() && Allowed[PRegOpt] != DstReg)
437 if (PRegOpt < Allowed.size()) {
439 NewCosts[PRegOpt + 1] -= CBenefit;
445 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed1 =
447 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed2 =
453 Allowed2->size() + 1, 0);
454 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
455 G.
addEdge(N1Id, N2Id, std::move(Costs));
462 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
472 void addVirtRegCoalesce(
474 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed1,
475 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed2,
477 assert(CostMat.getRows() == Allowed1.size() + 1 &&
"Size mismatch.");
478 assert(CostMat.getCols() == Allowed2.size() + 1 &&
"Size mismatch.");
479 for (
unsigned I = 0; I != Allowed1.size(); ++
I) {
480 unsigned PReg1 = Allowed1[
I];
481 for (
unsigned J = 0; J != Allowed2.size(); ++J) {
482 unsigned PReg2 = Allowed2[J];
484 CostMat[I + 1][J + 1] -= Benefit;
495 void PBQPRAConstraint::anchor() {}
496 void PBQPRAConstraintList::anchor() {}
498 void RegAllocPBQP::getAnalysisUsage(
AnalysisUsage &au)
const {
537 VRegsToAlloc.insert(LI.
reg);
539 EmptyIntervalVRegs.insert(LI.
reg);
547 for (
unsigned i = 0; CSR[i] != 0; ++i)
560 *G.
getMetadata().MF.getSubtarget().getRegisterInfo();
562 std::vector<unsigned> Worklist(VRegsToAlloc.begin(), VRegsToAlloc.end());
564 while (!Worklist.empty()) {
565 unsigned VReg = Worklist.back();
576 std::vector<unsigned> VRegAllowed;
578 for (
unsigned I = 0; I != RawPRegOrder.
size(); ++
I) {
579 unsigned PReg = RawPRegOrder[
I];
584 if (!RegMaskOverlaps.
empty() && !RegMaskOverlaps.
test(PReg))
588 bool Interference =
false;
599 VRegAllowed.push_back(PReg);
604 if (VRegAllowed.empty()) {
606 spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
607 Worklist.insert(Worklist.end(), NewVRegs.
begin(), NewVRegs.
end());
615 for (
unsigned i = 0; i != VRegAllowed.size(); ++i)
617 NodeCosts[1 + i] += 1.0;
622 G.
getMetadata().getAllowedRegs(std::move(VRegAllowed)));
627 void RegAllocPBQP::spillVReg(
unsigned VReg,
632 VRegsToAlloc.erase(VReg);
634 VRegSpiller.
spill(LRE);
639 << LRE.getParent().weight <<
", New vregs: ");
646 assert(!LI.
empty() &&
"Empty spill range.");
648 VRegsToAlloc.insert(LI.
reg);
654 bool RegAllocPBQP::mapPBQPToRegAlloc(
const PBQPRAGraph &G,
664 bool AnotherRoundNeeded =
false;
676 unsigned PReg = G.
getNodeMetadata(NId).getAllowedRegs()[AllocOption - 1];
679 assert(PReg != 0 &&
"Invalid preg selected.");
685 spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
686 AnotherRoundNeeded |= !NewVRegs.
empty();
690 return !AnotherRoundNeeded;
699 for (RegSet::const_iterator
700 I = EmptyIntervalVRegs.begin(), E = EmptyIntervalVRegs.end();
725 getAnalysis<MachineBlockFrequencyInfo>();
748 findVRegIntervalsToAlloc(MF, LIS);
752 std::string FullyQualifiedName =
757 if (!VRegsToAlloc.empty()) {
760 std::unique_ptr<PBQPRAConstraintList> ConstraintsRoot =
761 llvm::make_unique<PBQPRAConstraintList>();
762 ConstraintsRoot->addConstraint(llvm::make_unique<SpillCosts>());
763 ConstraintsRoot->addConstraint(llvm::make_unique<Interference>());
765 ConstraintsRoot->addConstraint(llvm::make_unique<Coalescing>());
768 bool PBQPAllocComplete =
false;
771 while (!PBQPAllocComplete) {
772 DEBUG(
dbgs() <<
" PBQP Regalloc round " << Round <<
":\n");
775 initializeGraph(G, VRM, *VRegSpiller);
776 ConstraintsRoot->apply(G);
780 std::ostringstream RS;
782 std::string GraphFileName = FullyQualifiedName +
"." + RS.str() +
786 DEBUG(
dbgs() <<
"Dumping graph for round " << Round <<
" to \""
787 << GraphFileName <<
"\"\n");
793 PBQPAllocComplete = mapPBQPToRegAlloc(G, Solution, VRM, *VRegSpiller);
799 finalizeAlloc(MF, LIS, VRM);
800 VRegsToAlloc.clear();
801 EmptyIntervalVRegs.clear();
803 DEBUG(
dbgs() <<
"Post alloc VirtRegMap:\n" << VRM <<
"\n");
810 class PrintNodeInfo {
815 PrintNodeInfo(NodeId NId,
const Graph &G) : G(G), NId(NId) {}
822 OS << NId <<
" (" << RegClassName <<
':' <<
PrintReg(VReg, TRI) <<
')';
839 assert(Costs.getLength() != 0 &&
"Empty vector in graph.");
840 OS << PrintNodeInfo(NId, *
this) <<
": " << Costs <<
'\n';
847 assert(N1Id != N2Id &&
"PBQP graphs should not have self-edges.");
849 assert(M.getRows() != 0 &&
"No rows in matrix.");
850 assert(M.getCols() != 0 &&
"No cols in matrix.");
851 OS << PrintNodeInfo(N1Id, *
this) <<
' ' << M.getRows() <<
" rows / ";
852 OS << PrintNodeInfo(N2Id, *
this) <<
' ' << M.getCols() <<
" cols:\n";
861 for (
auto NId : nodeIds()) {
862 OS <<
" node" << NId <<
" [ label=\""
863 << PrintNodeInfo(NId, *
this) <<
"\\n"
864 << getNodeCosts(NId) <<
"\" ]\n";
867 OS <<
" edge [ len=" << nodeIds().size() <<
" ]\n";
868 for (
auto EId : edgeIds()) {
869 OS <<
" node" << getEdgeNode1Id(EId)
870 <<
" -- node" << getEdgeNode2Id(EId)
872 const Matrix &EdgeCosts = getEdgeCosts(EId);
873 for (
unsigned i = 0; i < EdgeCosts.getRows(); ++i) {
874 OS << EdgeCosts.getRowAsVector(i) <<
"\\n";
882 return new RegAllocPBQP(customPassID);
Represents a solution to a PBQP problem.
Abstract base for classes implementing PBQP register allocation constraints (e.g. ...
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
NodeId getEdgeNode1Id(EdgeId EId) const
Get the first node connected to this edge.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
static unsigned index2VirtReg(unsigned Index)
index2VirtReg - Convert a 0-based index to a virtual register number.
EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs)
Add an edge between the given nodes with the given costs.
unsigned getSimpleHint(unsigned VReg) const
getSimpleHint - Return the preferred register allocation hint, or 0 if a standard simple hint (Type =...
Holds a vector of the allowed physical regs for a vreg.
static RegisterRegAlloc RegisterPBQPRepAlloc("pbqp","PBQP register allocator", createDefaultPBQPRegisterAllocator)
const T & front() const
front - Get the first element.
DenseSet - This implements a dense probed hash-table based set.
EdgeId findEdge(NodeId N1Id, NodeId N2Id)
Get the edge connecting two nodes.
LiveInterval - This class represents the liveness of a register, or stack slot.
virtual ~PBQPRAConstraint()=0
NodeId getEdgeNode2Id(EdgeId EId) const
Get the second node connected to this edge.
unsigned getSelection(GraphBase::NodeId nodeId) const
Get a node's selection.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
std::string str() const
str - Get the contents as an std::string.
void updateEdgeCosts(EdgeId EId, OtherMatrixT Costs)
Update an edge's cost matrix.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
NodeId addEdgeBypassingCostAllocator(NodeId N1Id, NodeId N2Id, OtherMatrixPtrT Costs)
Add an edge bypassing the cost allocator.
RegAllocSolverImpl::GraphMetadata GraphMetadata
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
void calculateSpillWeightsAndHints(LiveIntervals &LIS, MachineFunction &MF, const MachineLoopInfo &MLI, const MachineBlockFrequencyInfo &MBFI, VirtRegAuxInfo::NormalizingFn norm=normalizeSpillWeight)
Compute spill weights and allocation hints for all virtual register live intervals.
void assignVirt2Phys(unsigned virtReg, unsigned physReg)
creates a mapping for the specified virtual register to the specified physical register ...
bool checkRegMaskInterference(LiveInterval &LI, BitVector &UsableRegs)
checkRegMaskInterference - Test if LI is live across any register mask instructions, and compute a bit mask of physical registers that are not clobbered by any of them.
const Vector & getNodeCosts(NodeId NId) const
Get a node's cost vector.
BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const
getblockFreq - Return block frequency.
virtual void spill(LiveRangeEdit &LRE)=0
spill - Spill the LRE.getParent() live interval.
StringRef getName() const
Return a constant reference to the value's name.
FunctionPass * createPBQPRegisterAllocator(char *customPassID=nullptr)
Create a PBQP register allocator instance.
static const SCEV * apply(const SCEV *Scev, LoopToScevMapT &Map, ScalarEvolution &SE)
Applies the Map (Loop -> SCEV) to the given Scev.
RegAllocSolverImpl::RawMatrix RawMatrix
NodeMetadata & getNodeMetadata(NodeId NId)
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
AnalysisUsage & addRequired()
uint64_t getEntryFreq() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const TargetRegisterInfo * getTargetRegisterInfo() const
virtual std::unique_ptr< PBQPRAConstraint > getCustomPBQPConstraints() const
Return PBQPConstraint(s) for the target.
A helper class for register coalescers.
ArrayRef< MCPhysReg > getRawAllocationOrder(const MachineFunction &MF) const
getRawAllocationOrder - Returns the preferred order for allocating registers from this register class...
const TargetRegisterClass * getRegClass(unsigned Reg) const
getRegClass - Return the register class of the specified virtual register.
SmallVectorImpl< unsigned >::const_iterator iterator
Iterator for accessing the new registers added by this edit.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
Reg
All possible values of the reg field in the ModR/M byte.
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
const std::string & getModuleIdentifier() const
Get the module identifier which is, essentially, the name of the module.
void printDot(raw_ostream &OS) const
Print a representation of this graph in DOT format.
static cl::opt< bool > PBQPCoalescing("pbqp-coalescing", cl::desc("Attempt coalescing during PBQP register allocation."), cl::init(false), cl::Hidden)
PrintReg - Helper class for printing registers on a raw_ostream.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
const char * getRegClassName(const TargetRegisterClass *Class) const
getRegClassName - Returns the name of the register class.
RegAllocSolverImpl::Matrix Matrix
RegisterRegAlloc class - Track the registration of register allocators.
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
getCalleeSavedRegs - Return a null-terminated list of all of the callee saved registers on this targe...
EdgeIdSet edgeIds() const
size_t size() const
size - Get the array size.
NodeIdSet nodeIds() const
initializer< Ty > init(const Ty &Val)
void initializeSlotIndexesPass(PassRegistry &)
bool regsOverlap(unsigned regA, unsigned regB) const
regsOverlap - Returns true if the two registers are equal or alias each other.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
void initializeLiveStacksPass(PassRegistry &)
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool empty() const
empty - Tests whether there are no bits in this bitvector.
bool overlaps(const LiveRange &other) const
overlaps - Return true if the intersection of the two live ranges is not empty.
Represent the analysis usage information of a pass.
void initializeLiveIntervalsPass(PassRegistry &)
FunctionPass * createDefaultPBQPRegisterAllocator()
PBQPRegisterAllocation Pass - This pass implements the Partitioned Boolean Quadratic Prograaming (PBQ...
FunctionPass class - This class is used to implement most global optimizations.
static bool isACalleeSavedRegister(unsigned reg, const TargetRegisterInfo &TRI, const MachineFunction &MF)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
GraphMetadata & getMetadata()
Get a reference to the graph metadata.
AnalysisUsage & addRequiredID(const void *ID)
Module.h This file contains the declarations for the Module class.
bool test(unsigned Idx) const
const Matrix & getEdgeCosts(EdgeId EId) const
Get an edge's cost matrix.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LiveInterval & getInterval(unsigned Reg)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
The file should be opened in text mode on platforms that make this distinction.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
void initializeVirtRegMapPass(PassRegistry &)
Spiller * createInlineSpiller(MachineFunctionPass &pass, MachineFunction &mf, VirtRegMap &vrm)
Create and return a spiller that will insert spill code directly instead of deferring though VirtRegM...
static EdgeId invalidEdgeId()
Returns a value representing an invalid (non-existent) edge.
A raw_ostream that writes to a file descriptor.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Solution solve(PBQPRAGraph &G)
RegAllocSolverImpl::Vector Vector
unsigned getSpillOptionIdx()
Spill option index.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
static cl::opt< bool > PBQPDumpGraphs("pbqp-dump-graphs", cl::desc("Dump graphs for each function/round in the compilation unit."), cl::init(false), cl::Hidden)
static float normalizeSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr)
Normalize the spill weight of a live interval.
void clearAllVirt()
clears all virtual to physical register mappings
Module * getParent()
Get the module that this global value is contained inside of...
simple register Simple Register Coalescing
const MatrixPtr & getEdgeCostsPtr(EdgeId EId) const
Get a MatrixPtr to a node's cost matrix.
This class implements an extremely fast bulk output stream that can only output to a stream...
const char * getName(unsigned RegNo) const
Return the human-readable symbolic target-specific name for the specified physical register...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
void dump() const
Dump this graph to dbgs().
NodeId addNode(OtherVectorT Costs)
Add a node with the given costs.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
RegAllocSolverImpl::RawVector RawVector
SlotIndex - An opaque wrapper around machine indexes.
void setNodeCosts(NodeId NId, OtherVectorT Costs)
Set a node's cost vector.
static float normalizePBQPSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr)
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool reg_nodbg_empty(unsigned RegNo) const
reg_nodbg_empty - Return true if the only instructions using or defining Reg are Debug instructions...
LiveRange & getRegUnit(unsigned Unit)
getRegUnit - Return the live range for Unit.