19#define DEBUG_TYPE "machine-scheduler"
26 for (
const auto &
P : S1) {
28 if (
I == S2.
end() ||
I->second !=
P.second)
38unsigned GCNRegPressure::getRegKind(
Register Reg,
41 const auto RC =
MRI.getRegClass(Reg);
45 : STI->isAGPRClass(RC)
59 if (NewMask < PrevMask) {
64 switch (
auto Kind = getRegKind(Reg,
MRI)) {
74 assert(PrevMask < NewMask);
79 if (PrevMask.
none()) {
81 Value[Kind] += Sign *
MRI.getPressureSets(Reg).getWeight();
91 unsigned MaxOccupancy)
const {
92 const auto SGPROcc = std::min(MaxOccupancy,
95 std::min(MaxOccupancy,
96 ST.getOccupancyWithNumVGPRs(
getVGPRNum(ST.hasGFX90AInsts())));
97 const auto OtherSGPROcc = std::min(MaxOccupancy,
98 ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
99 const auto OtherVGPROcc =
100 std::min(MaxOccupancy,
101 ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts())));
103 const auto Occ = std::min(SGPROcc, VGPROcc);
104 const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);
106 return Occ > OtherOcc;
108 bool SGPRImportant = SGPROcc < VGPROcc;
109 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
112 if (SGPRImportant != OtherSGPRImportant) {
113 SGPRImportant =
false;
117 bool SGPRFirst = SGPRImportant;
118 for (
int I = 2;
I > 0; --
I, SGPRFirst = !SGPRFirst) {
121 auto OtherSW = O.getSGPRTuplesWeight();
126 auto OtherVW = O.getVGPRTuplesWeight();
131 return SGPRImportant ? (
getSGPRNum() < O.getSGPRNum()):
133 O.getVGPRNum(ST.hasGFX90AInsts()));
136#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
141 <<
"AGPRs: " << RP.getAGPRNum();
144 << ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()))
146 OS <<
", SGPRs: " << RP.getSGPRNum();
148 OS <<
"(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) <<
')';
149 OS <<
", LVGPR WT: " << RP.getVGPRTuplesWeight()
150 <<
", LSGPR WT: " << RP.getSGPRTuplesWeight();
152 OS <<
" -> Occ: " << RP.getOccupancy(*ST);
167 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.
getSubReg());
176 return MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(
SubReg);
178 auto MaxMask =
MRI.getMaxLaneMaskForVReg(MO.
getReg());
193 for (
const auto &MO :
MI.operands()) {
194 if (!MO.isReg() || !MO.getReg().isVirtual())
196 if (!MO.isUse() || !MO.readsReg())
201 auto Reg = MO.getReg();
205 I->LaneMask |= UsedMask;
221 if (LI.hasSubRanges()) {
222 for (
const auto &S : LI.subranges())
224 LiveMask |= S.LaneMask;
225 assert(LiveMask <
MRI.getMaxLaneMaskForVReg(Reg) ||
226 LiveMask ==
MRI.getMaxLaneMaskForVReg(Reg));
228 }
else if (LI.liveAt(
SI)) {
229 LiveMask =
MRI.getMaxLaneMaskForVReg(Reg);
238 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
244 LiveRegs[Reg] = LiveMask;
275 if (
MI.isDebugInstr())
282 for (
const auto &U : RegUses) {
283 auto LiveMask =
LiveRegs[U.RegUnit];
284 AtMIPressure.inc(U.RegUnit, LiveMask, LiveMask | U.LaneMask, *
MRI);
289 for (
const auto &MO :
MI.operands()) {
290 if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual() || MO.isDead())
293 auto Reg = MO.getReg();
297 auto &LiveMask =
I->second;
298 auto PrevMask = LiveMask;
299 LiveMask &= ~getDefRegMask(MO, *
MRI);
304 for (
const auto &U : RegUses) {
305 auto &LiveMask =
LiveRegs[U.RegUnit];
306 auto PrevMask = LiveMask;
307 LiveMask |= U.LaneMask;
315 MRI = &
MI.getParent()->getParent()->getRegInfo();
317 MBBEnd =
MI.getParent()->end();
320 if (NextMI == MBBEnd)
329 return NextMI == MBBEnd;
331 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
341 if (!MO.isReg() || !MO.getReg().isVirtual())
343 if (MO.isUse() && !MO.readsReg())
345 if (!SeenRegs.
insert(MO.getReg()).second)
357 auto PrevMask = It->second;
358 It->second &= ~S.LaneMask;
377 return NextMI == MBBEnd;
386 if (!MO.isReg() || !MO.isDef())
389 if (!Reg.isVirtual())
392 auto PrevMask = LiveMask;
401 if (NextMI == MBBEnd)
409 while (NextMI != End)
417 reset(*Begin, LiveRegsCopy);
421#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
427 for (
auto const &
P : TrackedLR) {
428 auto I = LISLR.
find(
P.first);
430 OS <<
" " << printReg(P.first, TRI) <<
":L" << PrintLaneMask(P.second)
431 <<
" isn't found in LIS reported set\n";
432 }
else if (
I->second !=
P.second) {
433 OS <<
" " << printReg(P.first, TRI)
434 <<
" masks doesn't match: LIS reported " << PrintLaneMask(I->second)
435 <<
", tracked " << PrintLaneMask(P.second) <<
'\n';
438 for (
auto const &
P : LISLR) {
439 auto I = TrackedLR.find(
P.first);
440 if (
I == TrackedLR.end()) {
441 OS <<
" " << printReg(P.first, TRI) <<
":L" << PrintLaneMask(P.second)
442 <<
" isn't found in tracked set\n";
453 if (!
isEqual(LISLR, TrackedLR)) {
454 dbgs() <<
"\nGCNUpwardRPTracker error: Tracked and"
455 " LIS reported livesets mismatch:\n"
463 dbgs() <<
"GCNUpwardRPTracker error: Pressure sets different\nTracked: "
475 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
476 Register Reg = Register::index2VirtReg(I);
477 auto It = LiveRegs.find(Reg);
478 if (It != LiveRegs.end() && It->second.any())
479 OS <<
' ' << printVRegOrUnit(Reg, TRI) <<
':'
480 << PrintLaneMask(It->second);
unsigned const MachineRegisterInfo * MRI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static LaneBitmask getDefRegMask(const MachineOperand &MO, const MachineRegisterInfo &MRI)
static LaneBitmask getUsedRegMask(const MachineOperand &MO, const MachineRegisterInfo &MRI, const LiveIntervals &LIS)
static SmallVector< RegisterMaskPair, 8 > collectVirtualRegUses(const MachineInstr &MI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
This file defines the GCNRegPressure class, which tracks registry pressure by bookkeeping number of S...
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
const MachineInstr * LastTrackedMI
GCNRegPressure CurPressure
GCNRegPressure MaxPressure
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, bool After)
const MachineRegisterInfo * MRI
const LiveIntervals & LIS
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
void recede(const MachineInstr &MI)
LiveInterval - This class represents the liveness of a register, or stack slot.
bool hasSubRanges() const
Returns true if subregister liveness information is available.
iterator_range< subrange_iterator > subranges()
bool hasInterval(Register Reg) const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
bool liveAt(SlotIndex index) const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Representation of each machine instruction.
iterator_range< mop_iterator > operands()
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterInfo * getTargetRegisterInfo() const
Simple wrapper around std::function<void(raw_ostream&)>.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static unsigned getNumCoveredRegs(LaneBitmask LM)
static bool isSGPRClass(const TargetRegisterClass *RC)
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getDeadSlot() const
Returns the dead def kill slot for the current instruction.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, const LiveIntervals &LIS)
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR, const GCNRPTracker::LiveRegSet &TrackedL, const TargetRegisterInfo *TRI)
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
unsigned getVGPRTuplesWeight() const
unsigned getVGPRNum(bool UnifiedVGPRFile) const
void inc(unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask, const MachineRegisterInfo &MRI)
unsigned getSGPRNum() const
bool less(const GCNSubtarget &ST, const GCNRegPressure &O, unsigned MaxOccupancy=std::numeric_limits< unsigned >::max()) const
unsigned getSGPRTuplesWeight() const
friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST)
constexpr bool none() const
static constexpr LaneBitmask getNone()