63 #define DEBUG_TYPE "regalloc"
71 cl::desc(
"Attempt coalescing during PBQP register allocation."),
77 cl::desc(
"Dump graphs for each function/round in the compilation unit."),
93 RegAllocPBQP(
char *cPassID =
nullptr)
102 StringRef getPassName()
const override {
return "PBQP Register Allocator"; }
117 typedef std::map<const LiveInterval*, unsigned> LI2NodeMap;
118 typedef std::vector<const LiveInterval*> Node2LIMap;
119 typedef std::vector<unsigned> AllowedSet;
120 typedef std::vector<AllowedSet> AllowedSetMap;
121 typedef std::pair<unsigned, unsigned> RegPair;
122 typedef std::map<RegPair, PBQP::PBQPNum> CoalesceMap;
123 typedef std::set<unsigned> RegSet;
127 RegSet VRegsToAlloc, EmptyIntervalVRegs;
176 if (SpillCost == 0.0)
179 SpillCost += MinSpillCost;
192 typedef std::pair<AllowedRegVecPtr, AllowedRegVecPtr> IKey;
195 typedef std::pair<PBQP::GraphBase::NodeId, PBQP::GraphBase::NodeId> IEdgeKey;
200 const DisjointAllowedRegsCache &
D)
const {
208 return D.count(IKey(NRegs, MRegs)) > 0;
210 return D.count(IKey(MRegs, NRegs)) > 0;
215 DisjointAllowedRegsCache &D) {
219 assert(NRegs != MRegs &&
"AllowedRegs can not be disjoint with itself");
222 D.insert(IKey(NRegs, MRegs));
224 D.insert(IKey(MRegs, NRegs));
231 typedef std::tuple<LiveInterval*, size_t, PBQP::GraphBase::NodeId>
234 static SlotIndex getStartPoint(
const IntervalInfo &
I) {
235 return std::get<0>(
I)->segments[std::get<1>(I)].start;
238 static SlotIndex getEndPoint(
const IntervalInfo &
I) {
239 return std::get<0>(
I)->segments[std::get<1>(I)].end;
243 return std::get<2>(
I);
246 static bool lowestStartPoint(
const IntervalInfo &I1,
247 const IntervalInfo &I2) {
250 return getStartPoint(I1) > getStartPoint(I2);
253 static bool lowestEndPoint(
const IntervalInfo &I1,
254 const IntervalInfo &I2) {
267 return std::get<0>(I1)->reg < std::get<0>(I2)->reg;
270 static bool isAtLastSegment(
const IntervalInfo &
I) {
271 return std::get<1>(
I) == std::get<0>(I)->size() - 1;
274 static IntervalInfo nextSegment(
const IntervalInfo &I) {
275 return std::make_tuple(std::get<0>(I), std::get<1>(I) + 1, std::get<2>(I));
298 DisjointAllowedRegsCache
D;
300 typedef std::set<IntervalInfo, decltype(&lowestEndPoint)> IntervalSet;
301 typedef std::priority_queue<IntervalInfo, std::vector<IntervalInfo>,
302 decltype(&lowestStartPoint)> IntervalQueue;
303 IntervalSet Active(lowestEndPoint);
304 IntervalQueue Inactive(lowestStartPoint);
310 assert(!LI.
empty() &&
"PBQP graph contains node for empty interval");
311 Inactive.push(std::make_tuple(&LI, 0, NId));
314 while (!Inactive.empty()) {
317 IntervalInfo Cur = Inactive.top();
320 IntervalSet::iterator RetireItr = Active.begin();
321 while (RetireItr != Active.end() &&
322 (getEndPoint(*RetireItr) <= getStartPoint(Cur))) {
325 if (!isAtLastSegment(*RetireItr))
326 Inactive.push(nextSegment(*RetireItr));
330 Active.erase(Active.begin(), RetireItr);
334 Cur = Inactive.top();
340 for (
const auto &
A : Active) {
345 if (haveDisjointAllowedRegs(G, NId, MId, D))
349 IEdgeKey EK(
std::min(NId, MId), std::max(NId, MId));
354 if (!createInterferenceEdge(G, NId, MId, C))
355 setDisjointAllowedRegs(G, NId, MId, D);
377 *G.
getMetadata().MF.getSubtarget().getRegisterInfo();
382 IKey K(&NRegs, &MRegs);
383 IMatrixCache::iterator I = C.find(K);
390 bool NodesInterfere =
false;
391 for (
unsigned I = 0; I != NRegs.size(); ++
I) {
392 unsigned PRegN = NRegs[
I];
393 for (
unsigned J = 0; J != MRegs.size(); ++J) {
394 unsigned PRegM = MRegs[J];
396 M[I + 1][J + 1] = std::numeric_limits<PBQP::PBQPNum>::infinity();
397 NodesInterfere =
true;
422 for (
const auto &
MBB : MF) {
423 for (
const auto &
MI :
MBB) {
426 if (!
CP.setRegisters(&
MI) ||
CP.getSrcReg() ==
CP.getDstReg())
429 unsigned DstReg =
CP.getDstReg();
430 unsigned SrcReg =
CP.getSrcReg();
436 if (!MF.getRegInfo().isAllocatable(DstReg))
441 const PBQPRAGraph::NodeMetadata::AllowedRegVector &
Allowed =
444 unsigned PRegOpt = 0;
445 while (PRegOpt < Allowed.size() && Allowed[PRegOpt] != DstReg)
448 if (PRegOpt < Allowed.size()) {
450 NewCosts[PRegOpt + 1] -= CBenefit;
456 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed1 =
458 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed2 =
464 Allowed2->size() + 1, 0);
465 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
466 G.
addEdge(N1Id, N2Id, std::move(Costs));
473 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
483 void addVirtRegCoalesce(
485 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed1,
486 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed2,
488 assert(CostMat.getRows() == Allowed1.size() + 1 &&
"Size mismatch.");
489 assert(CostMat.getCols() == Allowed2.size() + 1 &&
"Size mismatch.");
490 for (
unsigned I = 0; I != Allowed1.size(); ++
I) {
491 unsigned PReg1 = Allowed1[
I];
492 for (
unsigned J = 0; J != Allowed2.size(); ++J) {
493 unsigned PReg2 = Allowed2[J];
495 CostMat[I + 1][J + 1] -= Benefit;
506 void PBQPRAConstraint::anchor() {}
507 void PBQPRAConstraintList::anchor() {}
509 void RegAllocPBQP::getAnalysisUsage(
AnalysisUsage &au)
const {
548 VRegsToAlloc.insert(LI.
reg);
550 EmptyIntervalVRegs.insert(LI.
reg);
558 for (
unsigned i = 0; CSR[
i] != 0; ++
i)
571 *G.
getMetadata().MF.getSubtarget().getRegisterInfo();
573 std::vector<unsigned> Worklist(VRegsToAlloc.begin(), VRegsToAlloc.end());
575 while (!Worklist.empty()) {
576 unsigned VReg = Worklist.back();
587 std::vector<unsigned> VRegAllowed;
589 for (
unsigned I = 0; I != RawPRegOrder.
size(); ++
I) {
590 unsigned PReg = RawPRegOrder[
I];
595 if (!RegMaskOverlaps.
empty() && !RegMaskOverlaps.
test(PReg))
599 bool Interference =
false;
610 VRegAllowed.push_back(PReg);
615 if (VRegAllowed.empty()) {
617 spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
618 Worklist.insert(Worklist.end(), NewVRegs.
begin(), NewVRegs.
end());
626 for (
unsigned i = 0;
i != VRegAllowed.size(); ++
i)
628 NodeCosts[1 + i] += 1.0;
633 G.
getMetadata().getAllowedRegs(std::move(VRegAllowed)));
638 void RegAllocPBQP::spillVReg(
unsigned VReg,
643 VRegsToAlloc.erase(VReg);
645 nullptr, &DeadRemats);
646 VRegSpiller.
spill(LRE);
651 << LRE.getParent().weight <<
", New vregs: ");
660 VRegsToAlloc.insert(LI.
reg);
666 bool RegAllocPBQP::mapPBQPToRegAlloc(
const PBQPRAGraph &G,
676 bool AnotherRoundNeeded =
false;
688 unsigned PReg = G.
getNodeMetadata(NId).getAllowedRegs()[AllocOption - 1];
691 assert(PReg != 0 &&
"Invalid preg selected.");
697 spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
698 AnotherRoundNeeded |= !NewVRegs.
empty();
702 return !AnotherRoundNeeded;
711 for (RegSet::const_iterator
712 I = EmptyIntervalVRegs.begin(),
E = EmptyIntervalVRegs.end();
730 for (
auto DeadInst : DeadRemats) {
732 DeadInst->eraseFromParent();
747 getAnalysis<MachineBlockFrequencyInfo>();
770 findVRegIntervalsToAlloc(MF, LIS);
774 std::string FullyQualifiedName =
779 if (!VRegsToAlloc.empty()) {
782 std::unique_ptr<PBQPRAConstraintList> ConstraintsRoot =
783 llvm::make_unique<PBQPRAConstraintList>();
784 ConstraintsRoot->addConstraint(llvm::make_unique<SpillCosts>());
785 ConstraintsRoot->addConstraint(llvm::make_unique<Interference>());
787 ConstraintsRoot->addConstraint(llvm::make_unique<Coalescing>());
790 bool PBQPAllocComplete =
false;
793 while (!PBQPAllocComplete) {
794 DEBUG(
dbgs() <<
" PBQP Regalloc round " << Round <<
":\n");
797 initializeGraph(G, VRM, *VRegSpiller);
798 ConstraintsRoot->apply(G);
802 std::ostringstream RS;
804 std::string GraphFileName = FullyQualifiedName +
"." + RS.str() +
808 DEBUG(
dbgs() <<
"Dumping graph for round " << Round <<
" to \""
809 << GraphFileName <<
"\"\n");
815 PBQPAllocComplete = mapPBQPToRegAlloc(G, Solution, VRM, *VRegSpiller);
821 finalizeAlloc(MF, LIS, VRM);
822 postOptimization(*VRegSpiller, LIS);
823 VRegsToAlloc.clear();
824 EmptyIntervalVRegs.clear();
826 DEBUG(
dbgs() <<
"Post alloc VirtRegMap:\n" << VRM <<
"\n");
839 OS << NId <<
" (" << RegClassName <<
':' <<
PrintReg(VReg, TRI) <<
')';
846 assert(Costs.getLength() != 0 &&
"Empty vector in graph.");
854 assert(N1Id != N2Id &&
"PBQP graphs should not have self-edges.");
856 assert(M.getRows() != 0 &&
"No rows in matrix.");
857 assert(M.getCols() != 0 &&
"No cols in matrix.");
858 OS <<
PrintNodeInfo(N1Id, *
this) <<
' ' << M.getRows() <<
" rows / ";
859 OS <<
PrintNodeInfo(N2Id, *
this) <<
' ' << M.getCols() <<
" cols:\n";
868 for (
auto NId : nodeIds()) {
869 OS <<
" node" << NId <<
" [ label=\""
871 << getNodeCosts(NId) <<
"\" ]\n";
874 OS <<
" edge [ len=" << nodeIds().size() <<
" ]\n";
875 for (
auto EId : edgeIds()) {
876 OS <<
" node" << getEdgeNode1Id(EId)
877 <<
" -- node" << getEdgeNode2Id(EId)
879 const Matrix &EdgeCosts = getEdgeCosts(EId);
880 for (
unsigned i = 0; i < EdgeCosts.getRows(); ++
i) {
881 OS << EdgeCosts.getRowAsVector(i) <<
"\\n";
889 return new RegAllocPBQP(customPassID);
Represents a solution to a PBQP problem.
static Printable PrintNodeInfo(PBQP::RegAlloc::PBQPRAGraph::NodeId NId, const PBQP::RegAlloc::PBQPRAGraph &G)
Create Printable object for node and register info.
Abstract base for classes implementing PBQP register allocation constraints (e.g. ...
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
NodeId getEdgeNode1Id(EdgeId EId) const
Get the first node connected to this edge.
void RemoveMachineInstrFromMaps(MachineInstr &MI)
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
static unsigned index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs)
Add an edge between the given nodes with the given costs.
unsigned getSimpleHint(unsigned VReg) const
getSimpleHint - Return the preferred register allocation hint, or 0 if a standard simple hint (Type =...
Holds a vector of the allowed physical regs for a vreg.
static RegisterRegAlloc RegisterPBQPRepAlloc("pbqp","PBQP register allocator", createDefaultPBQPRegisterAllocator)
const T & front() const
front - Get the first element.
Implements a dense probed hash-table based set.
EdgeId findEdge(NodeId N1Id, NodeId N2Id)
Get the edge connecting two nodes.
LiveInterval - This class represents the liveness of a register, or stack slot.
virtual ~PBQPRAConstraint()=0
NodeId getEdgeNode2Id(EdgeId EId) const
Get the second node connected to this edge.
unsigned getSelection(GraphBase::NodeId nodeId) const
Get a node's selection.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
void updateEdgeCosts(EdgeId EId, OtherMatrixT Costs)
Update an edge's cost matrix.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
NodeId addEdgeBypassingCostAllocator(NodeId N1Id, NodeId N2Id, OtherMatrixPtrT Costs)
Add an edge bypassing the cost allocator.
RegAllocSolverImpl::GraphMetadata GraphMetadata
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
void assignVirt2Phys(unsigned virtReg, unsigned physReg)
creates a mapping for the specified virtual register to the specified physical register ...
bool checkRegMaskInterference(LiveInterval &LI, BitVector &UsableRegs)
checkRegMaskInterference - Test if LI is live across any register mask instructions, and compute a bit mask of physical registers that are not clobbered by any of them.
const Vector & getNodeCosts(NodeId NId) const
Get a node's cost vector.
BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const
getblockFreq - Return block frequency.
virtual void spill(LiveRangeEdit &LRE)=0
spill - Spill the LRE.getParent() live interval.
StringRef getName() const
Return a constant reference to the value's name.
FunctionPass * createPBQPRegisterAllocator(char *customPassID=nullptr)
Create a PBQP register allocator instance.
void dump() const
Dump this graph to dbgs().
RegAllocSolverImpl::RawMatrix RawMatrix
NodeMetadata & getNodeMetadata(NodeId NId)
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
AnalysisUsage & addRequired()
uint64_t getEntryFreq() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const TargetRegisterInfo * getTargetRegisterInfo() const
virtual std::unique_ptr< PBQPRAConstraint > getCustomPBQPConstraints() const
Return PBQPConstraint(s) for the target.
A helper class for register coalescers.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
ArrayRef< MCPhysReg > getRawAllocationOrder(const MachineFunction &MF) const
Returns the preferred order for allocating registers from this register class in MF.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
SmallVectorImpl< unsigned >::const_iterator iterator
Iterator for accessing the new registers added by this edit.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
Reg
All possible values of the reg field in the ModR/M byte.
const std::string & getModuleIdentifier() const
Get the module identifier which is, essentially, the name of the module.
void printDot(raw_ostream &OS) const
Print a representation of this graph in DOT format.
static cl::opt< bool > PBQPCoalescing("pbqp-coalescing", cl::desc("Attempt coalescing during PBQP register allocation."), cl::init(false), cl::Hidden)
LLVM_NODISCARD bool empty() const
const char * getRegClassName(const TargetRegisterClass *Class) const
Returns the name of the register class.
RegAllocSolverImpl::Matrix Matrix
RegisterRegAlloc class - Track the registration of register allocators.
Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubRegIdx=0)
Prints virtual and physical registers with or without a TRI instance.
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
Return a null-terminated list of all of the callee-saved registers on this target.
EdgeIdSet edgeIds() const
size_t size() const
size - Get the array size.
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
NodeIdSet nodeIds() const
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
initializer< Ty > init(const Ty &Val)
void initializeSlotIndexesPass(PassRegistry &)
bool regsOverlap(unsigned regA, unsigned regB) const
Returns true if the two registers are equal or alias each other.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
void initializeLiveStacksPass(PassRegistry &)
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
bool empty() const
empty - Tests whether there are no bits in this bitvector.
bool overlaps(const LiveRange &other) const
overlaps - Return true if the intersection of the two live ranges is not empty.
Represent the analysis usage information of a pass.
void initializeLiveIntervalsPass(PassRegistry &)
FunctionPass * createDefaultPBQPRegisterAllocator()
PBQPRegisterAllocation Pass - This pass implements the Partitioned Boolean Quadratic Prograaming (PBQ...
FunctionPass class - This class is used to implement most global optimizations.
static bool isACalleeSavedRegister(unsigned reg, const TargetRegisterInfo &TRI, const MachineFunction &MF)
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
void apply(Opt *O, const Mod &M, const Mods &...Ms)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
GraphMetadata & getMetadata()
Get a reference to the graph metadata.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
void calculateSpillWeightsAndHints(LiveIntervals &LIS, MachineFunction &MF, VirtRegMap *VRM, const MachineLoopInfo &MLI, const MachineBlockFrequencyInfo &MBFI, VirtRegAuxInfo::NormalizingFn norm=normalizeSpillWeight)
Compute spill weights and allocation hints for all virtual register live intervals.
AnalysisUsage & addRequiredID(const void *ID)
Module.h This file contains the declarations for the Module class.
bool test(unsigned Idx) const
const Matrix & getEdgeCosts(EdgeId EId) const
Get an edge's cost matrix.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LiveInterval & getInterval(unsigned Reg)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
The file should be opened in text mode on platforms that make this distinction.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
TargetSubtargetInfo - Generic base class for all target subtargets.
void initializeVirtRegMapPass(PassRegistry &)
Spiller * createInlineSpiller(MachineFunctionPass &pass, MachineFunction &mf, VirtRegMap &vrm)
Create and return a spiller that will insert spill code directly instead of deferring though VirtRegM...
static EdgeId invalidEdgeId()
Returns a value representing an invalid (non-existent) edge.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
A raw_ostream that writes to a file descriptor.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Solution solve(PBQPRAGraph &G)
virtual void postOptimization()
RegAllocSolverImpl::Vector Vector
unsigned getSpillOptionIdx()
Spill option index.
static cl::opt< bool > PBQPDumpGraphs("pbqp-dump-graphs", cl::desc("Dump graphs for each function/round in the compilation unit."), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static float normalizeSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr)
Normalize the spill weight of a live interval.
void clearAllVirt()
clears all virtual to physical register mappings
Module * getParent()
Get the module that this global value is contained inside of...
simple register Simple Register Coalescing
const MatrixPtr & getEdgeCostsPtr(EdgeId EId) const
Get a MatrixPtr to a node's cost matrix.
This class implements an extremely fast bulk output stream that can only output to a stream...
const char * getName(unsigned RegNo) const
Return the human-readable symbolic target-specific name for the specified physical register...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
NodeId addNode(OtherVectorT Costs)
Add a node with the given costs.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Simple wrapper around std::function<void(raw_ostream&)>.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
RegAllocSolverImpl::RawVector RawVector
SlotIndex - An opaque wrapper around machine indexes.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
void setNodeCosts(NodeId NId, OtherVectorT Costs)
Set a node's cost vector.
static float normalizePBQPSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr)
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool reg_nodbg_empty(unsigned RegNo) const
reg_nodbg_empty - Return true if the only instructions using or defining Reg are Debug instructions...
LiveRange & getRegUnit(unsigned Unit)
getRegUnit - Return the live range for Unit.
Properties which a MachineFunction may have at a given point in time.