LLVM  6.0.0svn
GCNSchedStrategy.cpp
Go to the documentation of this file.
1 //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This contains a MachineSchedStrategy implementation for maximizing wave
12 /// occupancy on GCN hardware.
13 //===----------------------------------------------------------------------===//
14 
15 #include "GCNSchedStrategy.h"
16 #include "AMDGPUSubtarget.h"
17 #include "SIInstrInfo.h"
18 #include "SIMachineFunctionInfo.h"
19 #include "SIRegisterInfo.h"
22 
23 #define DEBUG_TYPE "machine-scheduler"
24 
25 using namespace llvm;
26 
28  const MachineSchedContext *C) :
29  GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
30 
31 static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs,
32  const MachineFunction &MF) {
33 
34  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
36  unsigned MinRegOccupancy = std::min(ST.getOccupancyWithNumSGPRs(SGPRs),
37  ST.getOccupancyWithNumVGPRs(VGPRs));
38  return std::min(MinRegOccupancy,
39  ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
40  *MF.getFunction()));
41 }
42 
45 
46  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
47 
48  MF = &DAG->MF;
49 
50  const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
51 
52  // FIXME: This is also necessary, because some passes that run after
53  // scheduling and before regalloc increase register pressure.
54  const int ErrorMargin = 3;
55 
56  SGPRExcessLimit = Context->RegClassInfo
57  ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
58  VGPRExcessLimit = Context->RegClassInfo
59  ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
60  if (TargetOccupancy) {
61  SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
62  VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
63  } else {
64  SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
65  SRI->getSGPRPressureSet());
66  VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
67  SRI->getVGPRPressureSet());
68  }
69 
70  SGPRCriticalLimit -= ErrorMargin;
71  VGPRCriticalLimit -= ErrorMargin;
72 }
73 
74 void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
75  bool AtTop, const RegPressureTracker &RPTracker,
76  const SIRegisterInfo *SRI,
77  unsigned SGPRPressure,
78  unsigned VGPRPressure) {
79 
80  Cand.SU = SU;
81  Cand.AtTop = AtTop;
82 
83  // getDownwardPressure() and getUpwardPressure() make temporary changes to
84  // the the tracker, so we need to pass those function a non-const copy.
85  RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
86 
87  std::vector<unsigned> Pressure;
88  std::vector<unsigned> MaxPressure;
89 
90  if (AtTop)
91  TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
92  else {
93  // FIXME: I think for bottom up scheduling, the register pressure is cached
94  // and can be retrieved by DAG->getPressureDif(SU).
95  TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
96  }
97 
98  unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
99  unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
100 
101  // If two instructions increase the pressure of different register sets
102  // by the same amount, the generic scheduler will prefer to schedule the
103  // instruction that increases the set with the least amount of registers,
104  // which in our case would be SGPRs. This is rarely what we want, so
105  // when we report excess/critical register pressure, we do it either
106  // only for VGPRs or only for SGPRs.
107 
108  // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
109  const unsigned MaxVGPRPressureInc = 16;
110  bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
111  bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
112 
113 
114  // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
115  // to increase the likelihood we don't go over the limits. We should improve
116  // the analysis to look through dependencies to find the path with the least
117  // register pressure.
118 
119  // We only need to update the RPDelata for instructions that increase
120  // register pressure. Instructions that decrease or keep reg pressure
121  // the same will be marked as RegExcess in tryCandidate() when they
122  // are compared with instructions that increase the register pressure.
123  if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
125  Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
126  }
127 
128  if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
130  Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
131  }
132 
133  // Register pressure is considered 'CRITICAL' if it is approaching a value
134  // that would reduce the wave occupancy for the execution unit. When
135  // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
136  // has the same cost, so we don't need to prefer one over the other.
137 
138  int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
139  int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
140 
141  if (SGPRDelta >= 0 || VGPRDelta >= 0) {
142  if (SGPRDelta > VGPRDelta) {
144  Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
145  } else {
147  Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
148  }
149  }
150 }
151 
152 // This function is mostly cut and pasted from
153 // GenericScheduler::pickNodeFromQueue()
154 void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
155  const CandPolicy &ZonePolicy,
156  const RegPressureTracker &RPTracker,
157  SchedCandidate &Cand) {
158  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
159  ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
160  unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
161  unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
162  ReadyQueue &Q = Zone.Available;
163  for (SUnit *SU : Q) {
164 
165  SchedCandidate TryCand(ZonePolicy);
166  initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
167  SGPRPressure, VGPRPressure);
168  // Pass SchedBoundary only when comparing nodes from the same boundary.
169  SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
170  GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
171  if (TryCand.Reason != NoCand) {
172  // Initialize resource delta if needed in case future heuristics query it.
173  if (TryCand.ResDelta == SchedResourceDelta())
174  TryCand.initResourceDelta(Zone.DAG, SchedModel);
175  Cand.setBest(TryCand);
176  }
177  }
178 }
179 
180 // This function is mostly cut and pasted from
181 // GenericScheduler::pickNodeBidirectional()
182 SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
183  // Schedule as far as possible in the direction of no choice. This is most
184  // efficient, but also provides the best heuristics for CriticalPSets.
185  if (SUnit *SU = Bot.pickOnlyChoice()) {
186  IsTopNode = false;
187  return SU;
188  }
189  if (SUnit *SU = Top.pickOnlyChoice()) {
190  IsTopNode = true;
191  return SU;
192  }
193  // Set the bottom-up policy based on the state of the current bottom zone and
194  // the instructions outside the zone, including the top zone.
195  CandPolicy BotPolicy;
196  setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
197  // Set the top-down policy based on the state of the current top zone and
198  // the instructions outside the zone, including the bottom zone.
199  CandPolicy TopPolicy;
200  setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
201 
202  // See if BotCand is still valid (because we previously scheduled from Top).
203  DEBUG(dbgs() << "Picking from Bot:\n");
204  if (!BotCand.isValid() || BotCand.SU->isScheduled ||
205  BotCand.Policy != BotPolicy) {
207  pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
208  assert(BotCand.Reason != NoCand && "failed to find the first candidate");
209  } else {
211  }
212 
213  // Check if the top Q has a better candidate.
214  DEBUG(dbgs() << "Picking from Top:\n");
215  if (!TopCand.isValid() || TopCand.SU->isScheduled ||
216  TopCand.Policy != TopPolicy) {
218  pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
219  assert(TopCand.Reason != NoCand && "failed to find the first candidate");
220  } else {
222  }
223 
224  // Pick best from BotCand and TopCand.
225  DEBUG(
226  dbgs() << "Top Cand: ";
228  dbgs() << "Bot Cand: ";
230  );
231  SchedCandidate Cand;
232  if (TopCand.Reason == BotCand.Reason) {
233  Cand = BotCand;
236  GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
237  if (TopCand.Reason != NoCand) {
238  Cand.setBest(TopCand);
239  } else {
240  TopCand.Reason = TopReason;
241  }
242  } else {
244  Cand = TopCand;
245  } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
246  Cand = BotCand;
247  } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
248  Cand = TopCand;
249  } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
250  Cand = BotCand;
251  } else {
252  if (BotCand.Reason > TopCand.Reason) {
253  Cand = TopCand;
254  } else {
255  Cand = BotCand;
256  }
257  }
258  }
259  DEBUG(
260  dbgs() << "Picking: ";
261  traceCandidate(Cand);
262  );
263 
264  IsTopNode = Cand.AtTop;
265  return Cand.SU;
266 }
267 
268 // This function is mostly cut and pasted from
269 // GenericScheduler::pickNode()
271  if (DAG->top() == DAG->bottom()) {
273  Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
274  return nullptr;
275  }
276  SUnit *SU;
277  do {
279  SU = Top.pickOnlyChoice();
280  if (!SU) {
281  CandPolicy NoPolicy;
282  TopCand.reset(NoPolicy);
283  pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
284  assert(TopCand.Reason != NoCand && "failed to find a candidate");
285  SU = TopCand.SU;
286  }
287  IsTopNode = true;
288  } else if (RegionPolicy.OnlyBottomUp) {
289  SU = Bot.pickOnlyChoice();
290  if (!SU) {
291  CandPolicy NoPolicy;
292  BotCand.reset(NoPolicy);
293  pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
294  assert(BotCand.Reason != NoCand && "failed to find a candidate");
295  SU = BotCand.SU;
296  }
297  IsTopNode = false;
298  } else {
299  SU = pickNodeBidirectional(IsTopNode);
300  }
301  } while (SU->isScheduled);
302 
303  if (SU->isTopReady())
304  Top.removeReady(SU);
305  if (SU->isBottomReady())
306  Bot.removeReady(SU);
307 
308  DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr());
309  return SU;
310 }
311 
313  std::unique_ptr<MachineSchedStrategy> S) :
314  ScheduleDAGMILive(C, std::move(S)),
315  ST(MF.getSubtarget<SISubtarget>()),
316  MFI(*MF.getInfo<SIMachineFunctionInfo>()),
317  StartingOccupancy(ST.getOccupancyWithLocalMemSize(MFI.getLDSSize(),
318  *MF.getFunction())),
319  MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
320 
321  DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
322 }
323 
325  if (Stage == 0) {
326  // Just record regions at the first pass.
327  Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
328  return;
329  }
330 
331  std::vector<MachineInstr*> Unsched;
332  Unsched.reserve(NumRegionInstrs);
333  for (auto &I : *this)
334  Unsched.push_back(&I);
335 
336  GCNRegPressure PressureBefore;
337  if (LIS) {
338  PressureBefore = Pressure[RegionIdx];
339 
340  DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
341  GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
342  dbgs() << "Region live-in pressure: ";
343  llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
344  dbgs() << "Region register pressure: ";
345  PressureBefore.print(dbgs()));
346  }
347 
349  Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
350 
351  if (!LIS)
352  return;
353 
354  // Check the results of scheduling.
356  auto PressureAfter = getRealRegPressure();
357 
358  DEBUG(dbgs() << "Pressure after scheduling: "; PressureAfter.print(dbgs()));
359 
360  if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
361  PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
362  Pressure[RegionIdx] = PressureAfter;
363  DEBUG(dbgs() << "Pressure in desired limits, done.\n");
364  return;
365  }
366  unsigned WavesAfter = getMaxWaves(PressureAfter.getSGPRNum(),
367  PressureAfter.getVGPRNum(), MF);
368  unsigned WavesBefore = getMaxWaves(PressureBefore.getSGPRNum(),
369  PressureBefore.getVGPRNum(), MF);
370  DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore <<
371  ", after " << WavesAfter << ".\n");
372 
373  // We could not keep current target occupancy because of the just scheduled
374  // region. Record new occupancy for next scheduling cycle.
375  unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
376  if (NewOccupancy < MinOccupancy) {
377  MinOccupancy = NewOccupancy;
378  DEBUG(dbgs() << "Occupancy lowered for the function to "
379  << MinOccupancy << ".\n");
380  }
381 
382  if (WavesAfter >= WavesBefore) {
383  Pressure[RegionIdx] = PressureAfter;
384  return;
385  }
386 
387  DEBUG(dbgs() << "Attempting to revert scheduling.\n");
389  for (MachineInstr *MI : Unsched) {
390  if (MI->getIterator() != RegionEnd) {
391  BB->remove(MI);
392  BB->insert(RegionEnd, MI);
393  LIS->handleMove(*MI, true);
394  }
395  // Reset read-undef flags and update them later.
396  for (auto &Op : MI->operands())
397  if (Op.isReg() && Op.isDef())
398  Op.setIsUndef(false);
399  RegisterOperands RegOpers;
400  RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
401  if (ShouldTrackLaneMasks) {
402  // Adjust liveness and add missing dead+read-undef flags.
403  SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
404  RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
405  } else {
406  // Adjust for missing dead-def flags.
407  RegOpers.detectDeadDefs(*MI, *LIS);
408  }
409  RegionEnd = MI->getIterator();
410  ++RegionEnd;
411  DEBUG(dbgs() << "Scheduling " << *MI);
412  }
413  RegionBegin = Unsched.front()->getIterator();
414  Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
415 
417 }
418 
419 GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
421  RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
422  return RPTracker.moveMaxPressure();
423 }
424 
425 void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
427 
428  // If the block has the only successor then live-ins of that successor are
429  // live-outs of the current block. We can reuse calculated live set if the
430  // successor will be sent to scheduling past current block.
431  const MachineBasicBlock *OnlySucc = nullptr;
432  if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
433  SlotIndexes *Ind = LIS->getSlotIndexes();
434  if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
435  OnlySucc = *MBB->succ_begin();
436  }
437 
438  // Scheduler sends regions from the end of the block upwards.
439  size_t CurRegion = RegionIdx;
440  for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
441  if (Regions[CurRegion].first->getParent() != MBB)
442  break;
443  --CurRegion;
444 
445  auto I = MBB->begin();
446  auto LiveInIt = MBBLiveIns.find(MBB);
447  if (LiveInIt != MBBLiveIns.end()) {
448  auto LiveIn = std::move(LiveInIt->second);
449  RPTracker.reset(*MBB->begin(), &LiveIn);
450  MBBLiveIns.erase(LiveInIt);
451  } else {
452  I = Regions[CurRegion].first;
453  RPTracker.reset(*I);
454  }
455 
456  for ( ; ; ) {
457  I = RPTracker.getNext();
458 
459  if (Regions[CurRegion].first == I) {
460  LiveIns[CurRegion] = RPTracker.getLiveRegs();
461  RPTracker.clearMaxPressure();
462  }
463 
464  if (Regions[CurRegion].second == I) {
465  Pressure[CurRegion] = RPTracker.moveMaxPressure();
466  if (CurRegion-- == RegionIdx)
467  break;
468  }
469  RPTracker.advanceToNext();
470  RPTracker.advanceBeforeNext();
471  }
472 
473  if (OnlySucc) {
474  if (I != MBB->end()) {
475  RPTracker.advanceToNext();
476  RPTracker.advance(MBB->end());
477  }
478  RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
479  RPTracker.advanceBeforeNext();
480  MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
481  }
482 }
483 
486  DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
487 
488  LiveIns.resize(Regions.size());
489  Pressure.resize(Regions.size());
490 
491  do {
492  Stage++;
493  RegionIdx = 0;
494  MachineBasicBlock *MBB = nullptr;
495 
496  if (Stage > 1) {
497  // Retry function scheduling if we found resulting occupancy and it is
498  // lower than used for first pass scheduling. This will give more freedom
499  // to schedule low register pressure blocks.
500  // Code is partially copied from MachineSchedulerBase::scheduleRegions().
501 
502  if (!LIS || StartingOccupancy <= MinOccupancy)
503  break;
504 
505  DEBUG(dbgs()
506  << "Retrying function scheduling with lowest recorded occupancy "
507  << MinOccupancy << ".\n");
508 
509  S.setTargetOccupancy(MinOccupancy);
510  }
511 
512  for (auto Region : Regions) {
513  RegionBegin = Region.first;
514  RegionEnd = Region.second;
515 
516  if (RegionBegin->getParent() != MBB) {
517  if (MBB) finishBlock();
518  MBB = RegionBegin->getParent();
519  startBlock(MBB);
520  if (Stage == 1)
521  computeBlockPressure(MBB);
522  }
523 
524  unsigned NumRegionInstrs = std::distance(begin(), end());
525  enterRegion(MBB, begin(), end(), NumRegionInstrs);
526 
527  // Skip empty scheduling regions (0 or 1 schedulable instructions).
528  if (begin() == end() || begin() == std::prev(end())) {
529  exitRegion();
530  continue;
531  }
532 
533  DEBUG(dbgs() << "********** MI Scheduling **********\n");
534  DEBUG(dbgs() << MF.getName()
535  << ":BB#" << MBB->getNumber() << " " << MBB->getName()
536  << "\n From: " << *begin() << " To: ";
537  if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
538  else dbgs() << "End";
539  dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
540 
541  schedule();
542 
543  exitRegion();
544  ++RegionIdx;
545  }
546  finishBlock();
547 
548  } while (Stage < 2);
549 }
uint64_t CallInst * C
Interface definition for SIRegisterInfo.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
AMDGPU specific subclass of TargetSubtarget.
void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS)
Use liveness information to find dead defs not marked with a dead flag and move them to the DeadDefs ...
Each Scheduling boundary is associated with ready queues.
static void printLiveRegs(raw_ostream &OS, const LiveRegSet &LiveRegs, const MachineRegisterInfo &MRI)
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
const MachineSchedContext * Context
This is a minimal scheduler strategy.
decltype(MaxPressure) moveMaxPressure()
unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const
Inverse of getMaxLocalMemWithWaveCount.
void startBlock(MachineBasicBlock *bb) override
Prepares to perform scheduling in the given block.
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
void setUnitInc(int Inc)
void traceCandidate(const SchedCandidate &Cand)
unsigned second
void reset(const CandPolicy &NewPolicy)
ScheduleDAGMI * DAG
bool isBottomReady() const
Definition: ScheduleDAG.h:454
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
std::unique_ptr< MachineSchedStrategy > SchedImpl
decltype(LiveRegs) moveLiveRegs()
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
MachineFunction & MF
Machine function.
Definition: ScheduleDAG.h:570
bool isScheduled
True once scheduled.
Definition: ScheduleDAG.h:289
const RegPressureTracker & getBotRPTracker() const
decltype(LiveRegs) const & getLiveRegs() const
Definition: BitVector.h:920
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
SlotIndexes pass.
Definition: SlotIndexes.h:331
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def...
Definition: SlotIndexes.h:255
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
MachineBasicBlock::iterator top() const
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const
MachineSchedPolicy RegionPolicy
MachineBasicBlock::iterator begin() const
Returns an iterator to the top of the current scheduling region.
bool isTopReady() const
Definition: ScheduleDAG.h:451
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they&#39;re not in a MachineFuncti...
List of registers defined and used by a machine instruction.
const MachineBasicBlock::const_iterator getNext() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void print(raw_ostream &OS, const SISubtarget *ST=nullptr) const
RegisterClassInfo * RegClassInfo
CandReason
Represent the type of SchedCandidate found within a single queue.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:378
Helpers for implementing custom MachineSchedStrategy classes.
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
GCNScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
unsigned getSGPRNum() const
static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs, const MachineFunction &MF)
SlotIndex getMBBStartIdx(unsigned Num) const
Returns the first index in the given basic block number.
Definition: SlotIndexes.h:487
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone)
Apply a set of heursitics to a new candidate.
Track the current register pressure at some position in the instruction stream, and remember the high...
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
unsigned getSGPRPressureSet() const
Policy for scheduling the next instruction in the candidate&#39;s zone.
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
const TargetSchedModel * SchedModel
virtual void exitRegion()
Called when the scheduler has finished scheduling the current region.
const TargetRegisterInfo * TRI
unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const
Return the maximum number of waves per SIMD for kernels using VGPRs VGPRs.
ScheduleDAGMILive * DAG
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
unsigned first
unsigned getVGPRPressureSet() const
const std::vector< unsigned > & getRegSetPressureAtPos() const
Get the register set pressure at the current position, which may be less than the pressure across the...
void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the Regi...
GenericScheduler shrinks the unscheduled zone using heuristics to balance the schedule.
unsigned getRegPressureSetLimit(const MachineFunction &MF, unsigned Idx) const override
unsigned getVGPRNum() const
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
const RegPressureTracker & getTopRPTracker() const
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const
Return the maximum number of waves per SIMD for kernels using SGPRs SGPRs.
PressureChange CriticalMax
unsigned succ_size() const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
Representation of each machine instruction.
Definition: MachineInstr.h:59
LiveIntervals * LIS
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
const TargetRegisterInfo * TRI
Target processor register info.
Definition: ScheduleDAG.h:569
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
Status of an instruction&#39;s critical resource consumption.
SchedCandidate BotCand
Candidate last picked from Bot boundary.
MachineBasicBlock::iterator bottom() const
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
#define I(x, y, z)
Definition: MD5.cpp:58
Capture a change in pressure for a single pressure set.
void getUpwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction bottom-up.
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
SlotIndexes * getSlotIndexes() const
unsigned NodeNum
Entry # of node in the node vector.
Definition: ScheduleDAG.h:269
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
MachineBasicBlock::iterator end() const
Returns an iterator to the bottom of the current scheduling region.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
GCNMaxOccupancySchedStrategy(const MachineSchedContext *C)
SchedCandidate TopCand
Candidate last picked from Top boundary.
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
#define DEBUG(X)
Definition: Debug.h:118
void getDownwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction top-down.
MachineBasicBlock * BB
The block in which to insert instructions.
IRTranslator LLVM IR MI
bool empty() const
MachineRegisterInfo & MRI
Virtual/real register map.
Definition: ScheduleDAG.h:571
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:84
void initialize(ScheduleDAGMI *DAG) override
Initialize the strategy after building the DAG for a new region.
void finishBlock() override
Cleans up after scheduling in the given block.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
RegPressureTracker RPTracker
void handleMove(MachineInstr &MI, bool UpdateFlags=false)
Call this method to notify LiveIntervals that instruction MI has been moved within a basic block...
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:247