19#define DEBUG_TYPE "machine-scheduler"
26 for (
const auto &
P : S1) {
28 if (
I == S2.
end() ||
I->second !=
P.second)
38unsigned GCNRegPressure::getRegKind(
Register Reg,
41 const auto RC =
MRI.getRegClass(Reg);
45 : STI->isAGPRClass(RC)
59 if (NewMask < PrevMask) {
64 switch (
auto Kind = getRegKind(Reg,
MRI)) {
74 assert(PrevMask < NewMask);
79 if (PrevMask.
none()) {
83 Sign *
TRI->getRegClassWeight(
MRI.getRegClass(Reg)).RegWeight;
93 unsigned MaxOccupancy)
const {
94 const auto SGPROcc = std::min(MaxOccupancy,
97 std::min(MaxOccupancy,
98 ST.getOccupancyWithNumVGPRs(
getVGPRNum(ST.hasGFX90AInsts())));
99 const auto OtherSGPROcc = std::min(MaxOccupancy,
100 ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
101 const auto OtherVGPROcc =
102 std::min(MaxOccupancy,
103 ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts())));
105 const auto Occ = std::min(SGPROcc, VGPROcc);
106 const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);
108 return Occ > OtherOcc;
110 bool SGPRImportant = SGPROcc < VGPROcc;
111 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
114 if (SGPRImportant != OtherSGPRImportant) {
115 SGPRImportant =
false;
119 bool SGPRFirst = SGPRImportant;
120 for (
int I = 2;
I > 0; --
I, SGPRFirst = !SGPRFirst) {
123 auto OtherSW = O.getSGPRTuplesWeight();
128 auto OtherVW = O.getVGPRTuplesWeight();
133 return SGPRImportant ? (
getSGPRNum() < O.getSGPRNum()):
135 O.getVGPRNum(ST.hasGFX90AInsts()));
138#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
143 <<
"AGPRs: " << RP.getAGPRNum();
146 << ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()))
148 OS <<
", SGPRs: " << RP.getSGPRNum();
150 OS <<
"(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) <<
')';
151 OS <<
", LVGPR WT: " << RP.getVGPRTuplesWeight()
152 <<
", LSGPR WT: " << RP.getSGPRTuplesWeight();
154 OS <<
" -> Occ: " << RP.getOccupancy(*ST);
169 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.
getSubReg());
178 return MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(
SubReg);
180 auto MaxMask =
MRI.getMaxLaneMaskForVReg(MO.
getReg());
195 for (
const auto &MO :
MI.operands()) {
196 if (!MO.isReg() || !MO.getReg().isVirtual())
198 if (!MO.isUse() || !MO.readsReg())
203 auto Reg = MO.getReg();
207 I->LaneMask |= UsedMask;
223 if (LI.hasSubRanges()) {
224 for (
const auto &S : LI.subranges())
226 LiveMask |= S.LaneMask;
227 assert(LiveMask <
MRI.getMaxLaneMaskForVReg(Reg) ||
228 LiveMask ==
MRI.getMaxLaneMaskForVReg(Reg));
230 }
else if (LI.liveAt(SI)) {
231 LiveMask =
MRI.getMaxLaneMaskForVReg(Reg);
240 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
246 LiveRegs[Reg] = LiveMask;
277 if (
MI.isDebugInstr())
284 for (
const auto &U : RegUses) {
285 auto LiveMask =
LiveRegs[U.RegUnit];
286 AtMIPressure.inc(U.RegUnit, LiveMask, LiveMask | U.LaneMask, *
MRI);
291 for (
const auto &MO :
MI.all_defs()) {
292 if (!MO.getReg().isVirtual() || MO.isDead())
295 auto Reg = MO.getReg();
299 auto &LiveMask =
I->second;
300 auto PrevMask = LiveMask;
301 LiveMask &= ~getDefRegMask(MO, *
MRI);
306 for (
const auto &U : RegUses) {
307 auto &LiveMask =
LiveRegs[U.RegUnit];
308 auto PrevMask = LiveMask;
309 LiveMask |= U.LaneMask;
317 MRI = &
MI.getParent()->getParent()->getRegInfo();
319 MBBEnd =
MI.getParent()->end();
322 if (NextMI == MBBEnd)
331 return NextMI == MBBEnd;
333 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
343 if (!MO.isReg() || !MO.getReg().isVirtual())
345 if (MO.isUse() && !MO.readsReg())
347 if (!SeenRegs.
insert(MO.getReg()).second)
359 auto PrevMask = It->second;
360 It->second &= ~S.LaneMask;
366 }
else if (!LI.
liveAt(SI)) {
379 return NextMI == MBBEnd;
389 if (!Reg.isVirtual())
392 auto PrevMask = LiveMask;
401 if (NextMI == MBBEnd)
409 while (NextMI !=
End)
417 reset(*Begin, LiveRegsCopy);
421#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
427 for (
auto const &
P : TrackedLR) {
428 auto I = LISLR.
find(
P.first);
430 OS <<
" " << printReg(P.first, TRI) <<
":L" << PrintLaneMask(P.second)
431 <<
" isn't found in LIS reported set\n";
432 }
else if (
I->second !=
P.second) {
433 OS <<
" " << printReg(P.first, TRI)
434 <<
" masks doesn't match: LIS reported " << PrintLaneMask(I->second)
435 <<
", tracked " << PrintLaneMask(P.second) <<
'\n';
438 for (
auto const &
P : LISLR) {
439 auto I = TrackedLR.find(
P.first);
440 if (
I == TrackedLR.end()) {
441 OS <<
" " << printReg(P.first, TRI) <<
":L" << PrintLaneMask(P.second)
442 <<
" isn't found in tracked set\n";
453 if (!
isEqual(LISLR, TrackedLR)) {
454 dbgs() <<
"\nGCNUpwardRPTracker error: Tracked and"
455 " LIS reported livesets mismatch:\n"
463 dbgs() <<
"GCNUpwardRPTracker error: Pressure sets different\nTracked: "
475 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
476 Register Reg = Register::index2VirtReg(I);
477 auto It = LiveRegs.find(Reg);
478 if (It != LiveRegs.end() && It->second.any())
479 OS <<
' ' << printVRegOrUnit(Reg, TRI) <<
':'
480 << PrintLaneMask(It->second);
unsigned const MachineRegisterInfo * MRI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static LaneBitmask getDefRegMask(const MachineOperand &MO, const MachineRegisterInfo &MRI)
static LaneBitmask getUsedRegMask(const MachineOperand &MO, const MachineRegisterInfo &MRI, const LiveIntervals &LIS)
static SmallVector< RegisterMaskPair, 8 > collectVirtualRegUses(const MachineInstr &MI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
This file defines the GCNRegPressure class, which tracks registry pressure by bookkeeping number of S...
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
const MachineInstr * LastTrackedMI
GCNRegPressure CurPressure
GCNRegPressure MaxPressure
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, bool After)
const MachineRegisterInfo * MRI
const LiveIntervals & LIS
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
void recede(const MachineInstr &MI)
LiveInterval - This class represents the liveness of a register, or stack slot.
bool hasSubRanges() const
Returns true if subregister liveness information is available.
iterator_range< subrange_iterator > subranges()
bool hasInterval(Register Reg) const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
bool liveAt(SlotIndex index) const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Representation of each machine instruction.
iterator_range< mop_iterator > operands()
iterator_range< filtered_mop_iterator > all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterInfo * getTargetRegisterInfo() const
Simple wrapper around std::function<void(raw_ostream&)>.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static unsigned getNumCoveredRegs(LaneBitmask LM)
static bool isSGPRClass(const TargetRegisterClass *RC)
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getDeadSlot() const
Returns the dead def kill slot for the current instruction.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, const LiveIntervals &LIS)
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR, const GCNRPTracker::LiveRegSet &TrackedL, const TargetRegisterInfo *TRI)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
unsigned getVGPRTuplesWeight() const
unsigned getVGPRNum(bool UnifiedVGPRFile) const
void inc(unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask, const MachineRegisterInfo &MRI)
unsigned getSGPRNum() const
bool less(const GCNSubtarget &ST, const GCNRegPressure &O, unsigned MaxOccupancy=std::numeric_limits< unsigned >::max()) const
unsigned getSGPRTuplesWeight() const
friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST)
constexpr bool none() const
static constexpr LaneBitmask getNone()