31#define DEBUG_TYPE "machine-scheduler"
38 cl::desc(
"Disable unclustred high register pressure "
39 "reduction scheduling stage."),
44 "Sets the bias which adds weight to occupancy vs latency. Set it to "
45 "100 to chase the occupancy only."),
50 cl::desc(
"Relax occupancy targets for kernels which are memory "
51 "bound (amdgpu-membound-threshold), or "
52 "Wave Limited (amdgpu-limit-wave-threshold)."),
59 HasHighPressure(
false) {}
92 "VGPRCriticalLimit calculation method.\n");
97 VGPRBudget = std::max(VGPRBudget, Granule);
117 unsigned SGPRPressure,
118 unsigned VGPRPressure) {
140 unsigned NewSGPRPressure =
Pressure[AMDGPU::RegisterPressureSets::SReg_32];
141 unsigned NewVGPRPressure =
Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
151 const unsigned MaxVGPRPressureInc = 16;
152 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >=
VGPRExcessLimit;
153 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >=
SGPRExcessLimit;
185 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
187 if (SGPRDelta > VGPRDelta) {
207 unsigned SGPRPressure = 0;
208 unsigned VGPRPressure = 0;
210 SGPRPressure =
Pressure[AMDGPU::RegisterPressureSets::SReg_32];
211 VGPRPressure =
Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
214 for (
SUnit *SU : Q) {
218 SGPRPressure, VGPRPressure);
269 "Last pick result should correspond to re-picking right now");
289 "Last pick result should correspond to re-picking right now");
305 IsTopNode = Cand.AtTop;
413 bool SameBoundary = Zone !=
nullptr;
446 const SUnit *CandNextClusterSU =
448 const SUnit *TryCandNextClusterSU =
451 Cand.
SU == CandNextClusterSU, TryCand, Cand,
Cluster))
481 StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy) {
483 LLVM_DEBUG(
dbgs() <<
"Starting occupancy is " << StartingOccupancy <<
".\n");
486 if (MinOccupancy != StartingOccupancy)
487 LLVM_DEBUG(
dbgs() <<
"Allowing Occupancy drops to " << MinOccupancy
492std::unique_ptr<GCNSchedStage>
494 switch (SchedStageID) {
496 return std::make_unique<OccInitialScheduleStage>(SchedStageID, *
this);
498 return std::make_unique<UnclusteredHighRPStage>(SchedStageID, *
this);
500 return std::make_unique<ClusteredLowOccStage>(SchedStageID, *
this);
502 return std::make_unique<PreRARematStage>(SchedStageID, *
this);
504 return std::make_unique<ILPInitialScheduleStage>(SchedStageID, *
this);
517GCNScheduleDAGMILive::getRealRegPressure(
unsigned RegionIdx)
const {
523void GCNScheduleDAGMILive::computeBlockPressure(
unsigned RegionIdx,
538 if (!Candidate->empty() && Candidate->pred_size() == 1) {
541 OnlySucc = Candidate;
546 size_t CurRegion = RegionIdx;
547 for (
size_t E = Regions.size(); CurRegion !=
E; ++CurRegion)
548 if (Regions[CurRegion].first->getParent() !=
MBB)
553 auto LiveInIt = MBBLiveIns.find(
MBB);
554 auto &Rgn = Regions[CurRegion];
556 if (LiveInIt != MBBLiveIns.end()) {
557 auto LiveIn = std::move(LiveInIt->second);
559 MBBLiveIns.erase(LiveInIt);
562 auto LRS = BBLiveInMap.lookup(NonDbgMI);
563#ifdef EXPENSIVE_CHECKS
572 if (Regions[CurRegion].first ==
I || NonDbgMI ==
I) {
573 LiveIns[CurRegion] =
RPTracker.getLiveRegs();
577 if (Regions[CurRegion].second ==
I) {
578 Pressure[CurRegion] =
RPTracker.moveMaxPressure();
579 if (CurRegion-- == RegionIdx)
592 MBBLiveIns[OnlySucc] =
RPTracker.moveLiveRegs();
597GCNScheduleDAGMILive::getBBLiveInMap()
const {
599 std::vector<MachineInstr *> BBStarters;
600 BBStarters.reserve(Regions.size());
601 auto I = Regions.rbegin(),
E = Regions.rend();
602 auto *
BB =
I->first->getParent();
605 BBStarters.push_back(
MI);
608 }
while (
I !=
E &&
I->first->getParent() ==
BB);
617 LiveIns.resize(Regions.size());
618 Pressure.resize(Regions.size());
619 RescheduleRegions.
resize(Regions.size());
620 RegionsWithHighRP.
resize(Regions.size());
621 RegionsWithExcessRP.
resize(Regions.size());
622 RegionsWithMinOcc.
resize(Regions.size());
623 RegionsWithIGLPInstrs.
resize(Regions.size());
624 RescheduleRegions.
set();
625 RegionsWithHighRP.
reset();
626 RegionsWithExcessRP.
reset();
627 RegionsWithMinOcc.
reset();
628 RegionsWithIGLPInstrs.
reset();
633void GCNScheduleDAGMILive::runSchedStages() {
634 LLVM_DEBUG(
dbgs() <<
"All regions recorded, starting actual scheduling.\n");
636 if (!Regions.empty())
637 BBLiveInMap = getBBLiveInMap();
642 if (!Stage->initGCNSchedStage())
645 for (
auto Region : Regions) {
649 if (!Stage->initGCNRegion()) {
650 Stage->advanceRegion();
656 Stage->finalizeGCNRegion();
659 Stage->finalizeGCNSchedStage();
667 OS <<
"Max Occupancy Initial Schedule";
670 OS <<
"Unclustered High Register Pressure Reschedule";
673 OS <<
"Clustered Low Occupancy Reschedule";
676 OS <<
"Pre-RA Rematerialize";
679 OS <<
"Max ILP Initial Schedule";
689 MFI(DAG.MFI), ST(DAG.ST), StageID(StageID) {}
706 if (
DAG.RegionsWithHighRP.
none() &&
DAG.RegionsWithExcessRP.
none())
712 InitialOccupancy =
DAG.MinOccupancy;
722 <<
"Retrying function scheduling without clustering. "
723 "Aggressivly try to reduce register pressure to achieve occupancy "
724 <<
DAG.MinOccupancy <<
".\n");
736 if (
DAG.StartingOccupancy <=
DAG.MinOccupancy)
740 dbgs() <<
"Retrying function scheduling with lowest recorded occupancy "
741 <<
DAG.MinOccupancy <<
".\n");
749 if (
DAG.RegionsWithMinOcc.
none() ||
DAG.Regions.size() == 1)
764 collectRematerializableInstructions();
765 if (RematerializableInsts.empty() || !sinkTriviallyRematInsts(
ST,
TII))
769 dbgs() <<
"Retrying function scheduling with improved occupancy of "
770 <<
DAG.MinOccupancy <<
" from rematerializing\n");
782 if (
DAG.MinOccupancy > InitialOccupancy) {
783 for (
unsigned IDX = 0; IDX <
DAG.Pressure.size(); ++IDX)
784 DAG.RegionsWithMinOcc[IDX] =
785 DAG.Pressure[IDX].getOccupancy(
DAG.ST) ==
DAG.MinOccupancy;
788 <<
" stage successfully increased occupancy to "
789 <<
DAG.MinOccupancy <<
'\n');
810 <<
"\n From: " << *
DAG.
begin() <<
" To: ";
812 else dbgs() <<
"End";
813 dbgs() <<
" RegionInstrs: " << NumRegionInstrs <<
'\n');
820 for (
auto &
I :
DAG) {
822 if (
I.getOpcode() == AMDGPU::SCHED_GROUP_BARRIER ||
823 I.getOpcode() == AMDGPU::IGLP_OPT)
834 dbgs() <<
"Pressure before scheduling:\nRegion live-ins:"
836 <<
"Region live-in pressure: "
857 DAG.MinOccupancy <= InitialOccupancy) &&
931 unsigned TargetOccupancy =
933 unsigned WavesAfter =
935 unsigned WavesBefore =
937 LLVM_DEBUG(
dbgs() <<
"Occupancy before scheduling: " << WavesBefore
938 <<
", after " << WavesAfter <<
".\n");
944 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
948 if (WavesAfter < WavesBefore && WavesAfter <
DAG.MinOccupancy &&
950 LLVM_DEBUG(
dbgs() <<
"Function is memory bound, allow occupancy drop up to "
952 NewOccupancy = WavesAfter;
955 if (NewOccupancy <
DAG.MinOccupancy) {
956 DAG.MinOccupancy = NewOccupancy;
960 <<
DAG.MinOccupancy <<
".\n");
988 unsigned ReadyCycle = CurrCycle;
989 for (
auto &
D : SU.
Preds) {
990 if (
D.isAssignedRegDep()) {
994 ReadyCycle = std::max(ReadyCycle, DefReady +
Latency);
997 ReadyCycles[SU.
NodeNum] = ReadyCycle;
1004 std::pair<MachineInstr *, unsigned>
B)
const {
1005 return A.second <
B.second;
1011 if (ReadyCycles.empty())
1013 unsigned BBNum = ReadyCycles.begin()->first->getParent()->getNumber();
1014 dbgs() <<
"\n################## Schedule time ReadyCycles for MBB : " << BBNum
1015 <<
" ##################\n# Cycle #\t\t\tInstruction "
1019 for (
auto &
I : ReadyCycles) {
1020 if (
I.second > IPrev + 1)
1021 dbgs() <<
"****************************** BUBBLE OF " <<
I.second - IPrev
1022 <<
" CYCLES DETECTED ******************************\n\n";
1023 dbgs() <<
"[ " <<
I.second <<
" ] : " << *
I.first <<
"\n";
1036 unsigned SumBubbles = 0;
1038 unsigned CurrCycle = 0;
1039 for (
auto &SU : InputSchedule) {
1040 unsigned ReadyCycle =
1042 SumBubbles += ReadyCycle - CurrCycle;
1044 ReadyCyclesSorted.insert(std::make_pair(SU.getInstr(), ReadyCycle));
1046 CurrCycle = ++ReadyCycle;
1069 unsigned SumBubbles = 0;
1071 unsigned CurrCycle = 0;
1072 for (
auto &
MI :
DAG) {
1076 unsigned ReadyCycle =
1078 SumBubbles += ReadyCycle - CurrCycle;
1080 ReadyCyclesSorted.insert(std::make_pair(SU->
getInstr(), ReadyCycle));
1082 CurrCycle = ++ReadyCycle;
1099 if (WavesAfter <
DAG.MinOccupancy)
1134 <<
"\n\t *** In shouldRevertScheduling ***\n"
1135 <<
" *********** BEFORE UnclusteredHighRPStage ***********\n");
1140 <<
"\n *********** AFTER UnclusteredHighRPStage ***********\n");
1142 unsigned OldMetric = MBefore.
getMetric();
1143 unsigned NewMetric = MAfter.
getMetric();
1144 unsigned WavesBefore =
1151 LLVM_DEBUG(
dbgs() <<
"\tMetric before " << MBefore <<
"\tMetric after "
1152 << MAfter <<
"Profit: " << Profit <<
"\n");
1190 LLVM_DEBUG(
dbgs() <<
"New pressure will result in more spilling.\n");
1205 int SkippedDebugInstr = 0;
1207 if (
MI->isDebugInstr()) {
1208 ++SkippedDebugInstr;
1215 if (!
MI->isDebugInstr())
1220 for (
auto &
Op :
MI->all_defs())
1221 Op.setIsUndef(
false);
1224 if (!
MI->isDebugInstr()) {
1242 while (SkippedDebugInstr-- > 0)
1251 if (
MI->isDebugInstr())
1265void PreRARematStage::collectRematerializableInstructions() {
1279 if (
Op->getSubReg() != 0 || !isTriviallyReMaterializable(*Def))
1283 if (Def->getParent() == UseI->
getParent())
1289 bool AddedToRematList =
false;
1290 for (
unsigned I = 0,
E =
DAG.Regions.size();
I !=
E; ++
I) {
1291 auto It =
DAG.LiveIns[
I].find(Reg);
1292 if (It !=
DAG.LiveIns[
I].end() && !It->second.none()) {
1293 if (
DAG.RegionsWithMinOcc[
I]) {
1294 RematerializableInsts[
I][Def] = UseI;
1295 AddedToRematList =
true;
1300 RematDefToLiveInRegions[Def].push_back(
I);
1303 if (!AddedToRematList)
1304 RematDefToLiveInRegions.erase(Def);
1308bool PreRARematStage::sinkTriviallyRematInsts(
const GCNSubtarget &ST,
1313 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>, 32>
1321 NewRescheduleRegions.
resize(
DAG.Regions.size());
1325 for (
const auto &It : RematDefToLiveInRegions)
1326 ImpactedRegions.
insert(It.second.begin(), It.second.end());
1330 for (
auto Idx : ImpactedRegions) {
1334 NewRegions =
DAG.Regions;
1335 NewRescheduleRegions.
reset();
1338 bool Improved =
false;
1339 for (
auto I : ImpactedRegions) {
1340 if (!
DAG.RegionsWithMinOcc[
I])
1345 int SGPRUsage = NewPressure[
I].getSGPRNum();
1354 if (NewPressure[
I].getOccupancy(
ST) >
DAG.MinOccupancy) {
1355 NewRescheduleRegions[
I] =
true;
1363 int TotalSinkableRegs = 0;
1364 for (
const auto &It : RematerializableInsts[
I]) {
1367 TotalSinkableRegs +=
1370 int VGPRsAfterSink = VGPRUsage - TotalSinkableRegs;
1374 if (OptimisticOccupancy <=
DAG.MinOccupancy)
1377 unsigned ImproveOccupancy = 0;
1379 for (
auto &It : RematerializableInsts[
I]) {
1388 TII->reMaterialize(*InsertPos->getParent(), InsertPos, Reg,
1389 Def->getOperand(0).getSubReg(), *Def, *
DAG.
TRI);
1394 InsertedMIToOldDef[NewMI] =
Def;
1398 DAG.updateRegionBoundaries(NewRegions, Def,
nullptr,
1402 DAG.updateRegionBoundaries(NewRegions, InsertPos, NewMI);
1408 for (
auto Idx : RematDefToLiveInRegions[Def]) {
1410 if (InsertPos->getParent() !=
DAG.Regions[
Idx].first->getParent()) {
1417 NewRegions[
Idx].first, NewRegions[
Idx].second);
1418 RPT.reset(*NonDbgMI, &NewLiveIns[
Idx]);
1419 RPT.advance(NewRegions[
Idx].second);
1420 NewPressure[
Idx] = RPT.moveMaxPressure();
1425 ImproveOccupancy = NewPressure[
I].getOccupancy(
ST);
1426 if (ImproveOccupancy >
DAG.MinOccupancy)
1431 for (
auto &Def : SinkedDefs)
1432 for (
auto TrackedIdx : RematDefToLiveInRegions[Def])
1433 RematerializableInsts[TrackedIdx].erase(Def);
1435 if (ImproveOccupancy <=
DAG.MinOccupancy)
1438 NewRescheduleRegions[
I] =
true;
1445 for (
auto &Entry : InsertedMIToOldDef) {
1450 MI->eraseFromParent();
1459 for (
auto &Entry : InsertedMIToOldDef) {
1464 DAG.BBLiveInMap.erase(OldMI);
1475 for (
auto Idx : ImpactedRegions) {
1478 DAG.MBBLiveIns.erase(
DAG.Regions[
Idx].first->getParent());
1480 DAG.Regions = NewRegions;
1481 DAG.RescheduleRegions = NewRescheduleRegions;
1490bool PreRARematStage::isTriviallyReMaterializable(
const MachineInstr &
MI) {
1495 if (MO.getReg().isVirtual())
1505void GCNScheduleDAGMILive::updateRegionBoundaries(
1509 unsigned I = 0,
E = RegionBoundaries.size();
1511 while (
I !=
E &&
MI->getParent() != RegionBoundaries[
I].first->getParent())
1514 for (;
I !=
E; ++
I) {
1515 if (
MI->getParent() != RegionBoundaries[
I].first->getParent())
1518 if (Removing &&
MI == RegionBoundaries[
I].first &&
1519 MI == RegionBoundaries[
I].second) {
1522 RegionBoundaries[
I] =
1523 std::pair(
MI->getParent()->end(),
MI->getParent()->end());
1526 if (
MI == RegionBoundaries[
I].first) {
1528 RegionBoundaries[
I] =
1529 std::pair(std::next(
MI), RegionBoundaries[
I].second);
1533 RegionBoundaries[
I].second);
1536 if (Removing &&
MI == RegionBoundaries[
I].second) {
1537 RegionBoundaries[
I] = std::pair(RegionBoundaries[
I].first, std::prev(
MI));
1546 unsigned Opc = MI->getOpcode();
1547 return Opc == AMDGPU::SCHED_GROUP_BARRIER || Opc == AMDGPU::IGLP_OPT;
1553 bool RemoveKillFlags)
1558 if (HasIGLPInstrs) {
1559 SavedMutations.clear();
MachineInstrBuilder MachineInstrBuilder & DefMI
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static cl::opt< bool > RelaxedOcc("amdgpu-schedule-relaxed-occupancy", cl::Hidden, cl::desc("Relax occupancy targets for kernels which are memory " "bound (amdgpu-membound-threshold), or " "Wave Limited (amdgpu-limit-wave-threshold)."), cl::init(false))
static void printScheduleModel(std::set< std::pair< MachineInstr *, unsigned >, EarlierIssuingCycle > &ReadyCycles)
static bool hasIGLPInstrs(ScheduleDAGInstrs *DAG)
static cl::opt< bool > DisableUnclusterHighRP("amdgpu-disable-unclustred-high-rp-reschedule", cl::Hidden, cl::desc("Disable unclustred high register pressure " "reduction scheduling stage."), cl::init(false))
static cl::opt< unsigned > ScheduleMetricBias("amdgpu-schedule-metric-bias", cl::Hidden, cl::desc("Sets the bias which adds weight to occupancy vs latency. Set it to " "100 to chase the occupancy only."), cl::init(10))
const HexagonInstrInfo * TII
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
uint32_t getLDSSize() const
unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const
Inverse of getMaxLocalMemWithWaveCount.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
bool none() const
none - Returns true if none of the bits are set.
bool initGCNSchedStage() override
bool shouldRevertScheduling(unsigned WavesAfter) override
bool initGCNRegion() override
This class represents an Operation in the Expression.
bool erase(const KeyT &Val)
GCNMaxILPSchedStrategy(const MachineSchedContext *C)
bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone) const override
Apply a set of heuristics to a new candidate.
GCNMaxOccupancySchedStrategy(const MachineSchedContext *C)
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
GCNPostScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S, bool RemoveKillFlags)
virtual bool initGCNRegion()
GCNRegPressure PressureBefore
bool isRegionWithExcessRP() const
bool mayCauseSpilling(unsigned WavesAfter)
ScheduleMetrics getScheduleMetrics(const std::vector< SUnit > &InputSchedule)
GCNScheduleDAGMILive & DAG
const GCNSchedStageID StageID
std::vector< MachineInstr * > Unsched
GCNRegPressure PressureAfter
SIMachineFunctionInfo & MFI
unsigned computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle, DenseMap< unsigned, unsigned > &ReadyCycles, const TargetSchedModel &SM)
virtual void finalizeGCNSchedStage()
virtual bool initGCNSchedStage()
virtual bool shouldRevertScheduling(unsigned WavesAfter)
std::vector< std::unique_ptr< ScheduleDAGMutation > > SavedMutations
GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
MachineBasicBlock * CurrentMBB
This is a minimal scheduler strategy.
const unsigned HighRPSGPRBias
GCNSchedStrategy(const MachineSchedContext *C)
SmallVector< GCNSchedStageID, 4 > SchedStages
void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy, const RegPressureTracker &RPTracker, SchedCandidate &Cand)
SUnit * pickNodeBidirectional(bool &IsTopNode)
unsigned SGPRCriticalLimit
std::vector< unsigned > MaxPressure
bool hasNextStage() const
GCNSchedStageID getCurrentStage()
SmallVectorImpl< GCNSchedStageID >::iterator CurrentStage
unsigned VGPRCriticalLimit
void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, const RegPressureTracker &RPTracker, const SIRegisterInfo *SRI, unsigned SGPRPressure, unsigned VGPRPressure)
std::vector< unsigned > Pressure
void initialize(ScheduleDAGMI *DAG) override
Initialize the strategy after building the DAG for a new region.
const unsigned HighRPVGPRBias
unsigned getTargetOccupancy()
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
GCNSchedStageID getNextStage() const
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
GCNScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S)
bool hasGFX90AInsts() const
unsigned computeOccupancy(const Function &F, unsigned LDSSize=0, unsigned NumSGPRs=0, unsigned NumVGPRs=0) const
Return occupancy for the given function.
const SIInstrInfo * getInstrInfo() const override
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const
unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const
Return the maximum number of waves per SIMD for kernels using VGPRs VGPRs.
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const
Return the maximum number of waves per SIMD for kernels using SGPRs SGPRs.
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
void traceCandidate(const SchedCandidate &Cand)
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
const TargetSchedModel * SchedModel
const MachineSchedContext * Context
const TargetRegisterInfo * TRI
GenericScheduler shrinks the unscheduled zone using heuristics to balance the schedule.
SchedCandidate BotCand
Candidate last picked from Bot boundary.
SchedCandidate TopCand
Candidate last picked from Top boundary.
MachineSchedPolicy RegionPolicy
virtual bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone) const
Apply a set of heuristics to a new candidate.
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
bool shouldRevertScheduling(unsigned WavesAfter) override
bool hasInterval(Register Reg) const
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
void handleMove(MachineInstr &MI, bool UpdateFlags=false)
Call this method to notify LiveIntervals that instruction MI has been moved within a basic block.
SlotIndexes * getSlotIndexes() const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
void RemoveMachineInstrFromMaps(MachineInstr &MI)
void removeInterval(Register Reg)
Interval removal.
LiveInterval & createAndComputeVirtRegInterval(Register Reg)
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
succ_iterator succ_begin()
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
unsigned succ_size() const
MachineInstrBundleIterator< MachineInstr > iterator
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
const MachineBasicBlock * getParent() const
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void clearRegisterDeads(Register Reg)
Clear all dead flags on operands defining register Reg.
MachineOperand class - Representation of each machine instruction operand.
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
MachineOperand * getOneDef(Register Reg) const
Returns the defining operand if there is exactly one operand defining the specified register,...
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
bool hasOneDef(Register RegNo) const
Return true if there is exactly one operand defining the specified register.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
bool shouldRevertScheduling(unsigned WavesAfter) override
bool shouldRevertScheduling(unsigned WavesAfter) override
bool initGCNRegion() override
bool initGCNSchedStage() override
Capture a change in pressure for a single pressure set.
Helpers for implementing custom MachineSchedStrategy classes.
Track the current register pressure at some position in the instruction stream, and remember the high...
void advance()
Advance across the current instruction.
void getDownwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction top-down.
const std::vector< unsigned > & getRegSetPressureAtPos() const
Get the register set pressure at the current position, which may be less than the pressure across the...
void getUpwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction bottom-up.
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
List of registers defined and used by a machine instruction.
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the Regi...
void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS)
Use liveness information to find dead defs not marked with a dead flag and move them to the DeadDefs ...
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
const TargetSchedModel & getSchedModel() const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
unsigned getOccupancy() const
void increaseOccupancy(const MachineFunction &MF, unsigned Limit)
unsigned getMaxWavesPerEU() const
unsigned getMinWavesPerEU() const
unsigned getMinAllowedOccupancy() const
void limitOccupancy(const MachineFunction &MF)
static unsigned getNumCoveredRegs(LaneBitmask LM)
static bool isVGPRClass(const TargetRegisterClass *RC)
Scheduling unit. This is a node in the scheduling DAG.
unsigned NodeNum
Entry # of node in the node vector.
bool isScheduled
True once scheduled.
bool isBottomReady() const
SmallVector< SDep, 4 > Preds
All sunit predecessors.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Each Scheduling boundary is associated with ready queues.
unsigned getLatencyStallCycles(SUnit *SU)
Get the difference between the given SUnit's ready time and the current cycle.
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
A ScheduleDAG for scheduling lists of MachineInstr.
MachineBasicBlock::iterator end() const
Returns an iterator to the bottom of the current scheduling region.
MachineBasicBlock * BB
The block in which to insert instructions.
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
virtual void finalizeSchedule()
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
MachineBasicBlock::iterator begin() const
Returns an iterator to the top of the current scheduling region.
SUnit * getSUnit(MachineInstr *MI) const
Returns an existing SUnit for this MI, or nullptr.
virtual void exitRegion()
Called when the scheduler has finished scheduling the current region.
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
bool ShouldTrackLaneMasks
const RegPressureTracker & getBotRPTracker() const
bool isTrackingPressure() const
Return true if register pressure tracking is enabled.
const RegPressureTracker & getTopRPTracker() const
RegPressureTracker RPTracker
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
void startBlock(MachineBasicBlock *bb) override
Prepares to perform scheduling in the given block.
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
MachineBasicBlock::iterator top() const
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
MachineBasicBlock::iterator bottom() const
void finishBlock() override
Cleans up after scheduling in the given block.
const SUnit * getNextClusterPred() const
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
const SUnit * getNextClusterSucc() const
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
MachineRegisterInfo & MRI
Virtual/real register map.
const TargetInstrInfo * TII
Target instruction information.
std::vector< SUnit > SUnits
The scheduling units.
const TargetRegisterInfo * TRI
Target processor register info.
MachineFunction & MF
Machine function.
static const unsigned ScaleFactor
unsigned getMetric() const
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
SlotIndex getMBBStartIdx(unsigned Num) const
Returns the first index in the given basic block number.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetInstrInfo - Interface to description of machine instruction set.
bool isTriviallyReMaterializable(const MachineInstr &MI) const
Return true if the instruction is trivially rematerializable, meaning it has no side effects and requ...
Provide an instruction scheduling machine model to CodeGen passes.
virtual const TargetInstrInfo * getInstrInfo() const
bool initGCNSchedStage() override
bool initGCNRegion() override
void finalizeGCNSchedStage() override
bool shouldRevertScheduling(unsigned WavesAfter) override
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI)
unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, std::optional< bool > EnableWavefrontSize32)
@ C
The default llvm calling convention, compatible with C.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
unsigned getWeakLeft(const SUnit *SU, bool isTop)
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
cl::opt< bool > VerifyScheduling
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
bool tryPressure(const PressureChange &TryP, const PressureChange &CandP, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason, const TargetRegisterInfo *TRI, const MachineFunction &MF)
@ UnclusteredHighRPReschedule
@ ClusteredLowOccupancyReschedule
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation()
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone)
bool tryGreater(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
DenseMap< MachineInstr *, GCNRPTracker::LiveRegSet > getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS)
creates a map MachineInstr -> LiveRegSet R - range of iterators on instructions After - upon entry or...
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
bool tryLess(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
Return true if this heuristic determines order.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
int biasPhysReg(const SUnit *SU, bool isTop)
Minimize physical register live ranges.
Implement std::hash so that hash_code can be used in STL containers.
bool operator()(std::pair< MachineInstr *, unsigned > A, std::pair< MachineInstr *, unsigned > B) const
unsigned getOccupancy(const GCNSubtarget &ST) const
unsigned getVGPRNum(bool UnifiedVGPRFile) const
unsigned getAGPRNum() const
unsigned getSGPRNum() const
bool less(const GCNSubtarget &ST, const GCNRegPressure &O, unsigned MaxOccupancy=std::numeric_limits< unsigned >::max()) const
Policy for scheduling the next instruction in the candidate's zone.
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
void setBest(SchedCandidate &Best)
void reset(const CandPolicy &NewPolicy)
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
SchedResourceDelta ResDelta
Status of an instruction's critical resource consumption.
unsigned DemandedResources
static constexpr LaneBitmask getNone()
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
RegisterClassInfo * RegClassInfo
PressureChange CriticalMax
PressureChange CurrentMax