LLVM  12.0.0git
GCNIterativeScheduler.cpp
Go to the documentation of this file.
1 //===- GCNIterativeScheduler.cpp ------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements the class GCNIterativeScheduler.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "GCNIterativeScheduler.h"
15 #include "AMDGPUSubtarget.h"
16 #include "GCNSchedStrategy.h"
17 #include "SIMachineFunctionInfo.h"
18 
19 using namespace llvm;
20 
21 #define DEBUG_TYPE "machine-scheduler"
22 
23 namespace llvm {
24 
25 std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
26  const ScheduleDAG &DAG);
27 
28  std::vector<const SUnit*> makeGCNILPScheduler(ArrayRef<const SUnit*> BotRoots,
29  const ScheduleDAG &DAG);
30 }
31 
32 // shim accessors for different order containers
34  return MI;
35 }
36 static inline MachineInstr *getMachineInstr(const SUnit *SU) {
37  return SU->getInstr();
38 }
39 static inline MachineInstr *getMachineInstr(const SUnit &SU) {
40  return SU.getInstr();
41 }
42 
43 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
45 static void printRegion(raw_ostream &OS,
48  const LiveIntervals *LIS,
49  unsigned MaxInstNum =
51  auto BB = Begin->getParent();
52  OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
53  << BB->getName() << ":\n";
54  auto I = Begin;
55  MaxInstNum = std::max(MaxInstNum, 1u);
56  for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
57  if (!I->isDebugInstr() && LIS)
58  OS << LIS->getInstructionIndex(*I);
59  OS << '\t' << *I;
60  }
61  if (I != End) {
62  OS << "\t...\n";
63  I = std::prev(End);
64  if (!I->isDebugInstr() && LIS)
65  OS << LIS->getInstructionIndex(*I);
66  OS << '\t' << *I;
67  }
68  if (End != BB->end()) { // print boundary inst if present
69  OS << "----\n";
70  if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
71  OS << *End;
72  }
73 }
74 
79  const LiveIntervals *LIS) {
80  const auto BB = Begin->getParent();
81  const auto &MRI = BB->getParent()->getRegInfo();
82 
83  const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
84  OS << "LIn RP: ";
85  getRegPressure(MRI, LiveIns).print(OS);
86 
87  const auto BottomMI = End == BB->end() ? std::prev(End) : End;
88  const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
89  OS << "LOt RP: ";
90  getRegPressure(MRI, LiveOuts).print(OS);
91 }
92 
95  const auto &ST = MF.getSubtarget<GCNSubtarget>();
96  for (const auto R : Regions) {
97  OS << "Region to schedule ";
98  printRegion(OS, R->Begin, R->End, LIS, 1);
99  printLivenessInfo(OS, R->Begin, R->End, LIS);
100  OS << "Max RP: ";
101  R->MaxPressure.print(OS, &ST);
102  }
103 }
104 
107  const Region *R,
108  const GCNRegPressure &RP) const {
109  OS << "\nAfter scheduling ";
110  printRegion(OS, R->Begin, R->End, LIS);
111  printSchedRP(OS, R->MaxPressure, RP);
112  OS << '\n';
113 }
114 
117  const GCNRegPressure &Before,
118  const GCNRegPressure &After) const {
119  const auto &ST = MF.getSubtarget<GCNSubtarget>();
120  OS << "RP before: ";
121  Before.print(OS, &ST);
122  OS << "RP after: ";
123  After.print(OS, &ST);
124 }
125 #endif
126 
127 // DAG builder helper
130  SmallVector<SUnit *, 8> TopRoots;
131 
132  SmallVector<SUnit*, 8> BotRoots;
133 public:
135  : Sch(_Sch) {
136  auto BB = R.Begin->getParent();
137  Sch.BaseClass::startBlock(BB);
138  Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
139 
140  Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
141  /*TrackLaneMask*/true);
143  Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
144  }
145 
147  Sch.BaseClass::exitRegion();
148  Sch.BaseClass::finishBlock();
149  }
150 
152  return TopRoots;
153  }
155  return BotRoots;
156  }
157 };
158 
161  Region &Rgn;
162  std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
163  GCNRegPressure SaveMaxRP;
164 
165 public:
167  MachineSchedStrategy &OverrideStrategy,
168  GCNIterativeScheduler &_Sch)
169  : Sch(_Sch)
170  , Rgn(R)
171  , SaveSchedImpl(std::move(_Sch.SchedImpl))
172  , SaveMaxRP(R.MaxPressure) {
173  Sch.SchedImpl.reset(&OverrideStrategy);
174  auto BB = R.Begin->getParent();
175  Sch.BaseClass::startBlock(BB);
176  Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
177  }
178 
180  Sch.BaseClass::exitRegion();
181  Sch.BaseClass::finishBlock();
182  Sch.SchedImpl.release();
183  Sch.SchedImpl = std::move(SaveSchedImpl);
184  }
185 
186  void schedule() {
187  assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
188  LLVM_DEBUG(dbgs() << "\nScheduling ";
189  printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
190  Sch.BaseClass::schedule();
191 
192  // Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
193  Sch.RegionEnd = Rgn.End;
194  //assert(Rgn.End == Sch.RegionEnd);
195  Rgn.Begin = Sch.RegionBegin;
196  Rgn.MaxPressure.clear();
197  }
198 
199  void restoreOrder() {
200  assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
201  // DAG SUnits are stored using original region's order
202  // so just use SUnits as the restoring schedule
203  Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
204  }
205 };
206 
207 namespace {
208 
209 // just a stub to make base class happy
210 class SchedStrategyStub : public MachineSchedStrategy {
211 public:
212  bool shouldTrackPressure() const override { return false; }
213  bool shouldTrackLaneMasks() const override { return false; }
214  void initialize(ScheduleDAGMI *DAG) override {}
215  SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
216  void schedNode(SUnit *SU, bool IsTopNode) override {}
217  void releaseTopNode(SUnit *SU) override {}
218  void releaseBottomNode(SUnit *SU) override {}
219 };
220 
221 } // end anonymous namespace
222 
224  StrategyKind S)
225  : BaseClass(C, std::make_unique<SchedStrategyStub>())
226  , Context(C)
227  , Strategy(S)
228  , UPTracker(*LIS) {
229 }
230 
231 // returns max pressure for a region
235  const {
236  // For the purpose of pressure tracking bottom inst of the region should
237  // be also processed. End is either BB end, BB terminator inst or sched
238  // boundary inst.
239  auto const BBEnd = Begin->getParent()->end();
240  auto const BottomMI = End == BBEnd ? std::prev(End) : End;
241 
242  // scheduleRegions walks bottom to top, so its likely we just get next
243  // instruction to track
244  auto AfterBottomMI = std::next(BottomMI);
245  if (AfterBottomMI == BBEnd ||
246  &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
247  UPTracker.reset(*BottomMI);
248  } else {
250  }
251 
252  for (auto I = BottomMI; I != Begin; --I)
253  UPTracker.recede(*I);
254 
255  UPTracker.recede(*Begin);
256 
258  (dbgs() << "Tracked region ",
259  printRegion(dbgs(), Begin, End, LIS), false));
260  return UPTracker.moveMaxPressure();
261 }
262 
263 // returns max pressure for a tentative schedule
264 template <typename Range> GCNRegPressure
266  Range &&Schedule) const {
267  auto const BBEnd = R.Begin->getParent()->end();
269  if (R.End != BBEnd) {
270  // R.End points to the boundary instruction but the
271  // schedule doesn't include it
272  RPTracker.reset(*R.End);
273  RPTracker.recede(*R.End);
274  } else {
275  // R.End doesn't point to the boundary instruction
276  RPTracker.reset(*std::prev(BBEnd));
277  }
278  for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
280  }
281  return RPTracker.moveMaxPressure();
282 }
283 
287  unsigned NumRegionInstrs) {
289  if (NumRegionInstrs > 2) {
290  Regions.push_back(
291  new (Alloc.Allocate())
292  Region { Begin, End, NumRegionInstrs,
293  getRegionPressure(Begin, End), nullptr });
294  }
295 }
296 
297 void GCNIterativeScheduler::schedule() { // overriden
298  // do nothing
300  if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
301  dbgs() << "Max RP: ";
302  Regions.back()->MaxPressure.print(
304  } dbgs()
305  << '\n';);
306 }
307 
309  if (Regions.empty())
310  return;
311  switch (Strategy) {
312  case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
313  case SCHEDULE_MINREGFORCED: scheduleMinReg(true); break;
315  case SCHEDULE_ILP: scheduleILP(false); break;
316  }
317 }
318 
319 // Detach schedule from SUnits and interleave it with debug values.
320 // Returned schedule becomes independent of DAG state.
321 std::vector<MachineInstr*>
323  std::vector<MachineInstr*> Res;
324  Res.reserve(Schedule.size() * 2);
325 
326  if (FirstDbgValue)
327  Res.push_back(FirstDbgValue);
328 
329  const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
330  for (auto SU : Schedule) {
331  Res.push_back(SU->getInstr());
332  const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
333  return P.second == SU->getInstr();
334  });
335  if (D != DbgE)
336  Res.push_back(D->first);
337  }
338  return Res;
339 }
340 
342  ScheduleRef Schedule,
343  const GCNRegPressure &MaxRP) {
344  R.BestSchedule.reset(
345  new TentativeSchedule{ detachSchedule(Schedule), MaxRP });
346 }
347 
349  assert(R.BestSchedule.get() && "No schedule specified");
350  scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
351  R.BestSchedule.reset();
352 }
353 
354 // minimal required region scheduler, works for ranges of SUnits*,
355 // SUnits or MachineIntrs*
356 template <typename Range>
358  const GCNRegPressure &MaxRP) {
359  assert(RegionBegin == R.Begin && RegionEnd == R.End);
360  assert(LIS != nullptr);
361 #ifndef NDEBUG
362  const auto SchedMaxRP = getSchedulePressure(R, Schedule);
363 #endif
364  auto BB = R.Begin->getParent();
365  auto Top = R.Begin;
366  for (const auto &I : Schedule) {
367  auto MI = getMachineInstr(I);
368  if (MI != &*Top) {
369  BB->remove(MI);
370  BB->insert(Top, MI);
371  if (!MI->isDebugInstr())
372  LIS->handleMove(*MI, true);
373  }
374  if (!MI->isDebugInstr()) {
375  // Reset read - undef flags and update them later.
376  for (auto &Op : MI->operands())
377  if (Op.isReg() && Op.isDef())
378  Op.setIsUndef(false);
379 
380  RegisterOperands RegOpers;
381  RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
382  /*IgnoreDead*/false);
383  // Adjust liveness and add missing dead+read-undef flags.
384  auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
385  RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
386  }
387  Top = std::next(MI->getIterator());
388  }
389  RegionBegin = getMachineInstr(Schedule.front());
390 
391  // Schedule consisting of MachineInstr* is considered 'detached'
392  // and already interleaved with debug values
393  if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
395  // Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
396  //assert(R.End == RegionEnd);
397  RegionEnd = R.End;
398  }
399 
400  R.Begin = RegionBegin;
401  R.MaxPressure = MaxRP;
402 
403 #ifndef NDEBUG
404  const auto RegionMaxRP = getRegionPressure(R);
405  const auto &ST = MF.getSubtarget<GCNSubtarget>();
406 #endif
407  assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP))
408  || (dbgs() << "Max RP mismatch!!!\n"
409  "RP for schedule (calculated): ",
410  SchedMaxRP.print(dbgs(), &ST),
411  dbgs() << "RP for schedule (reported): ",
412  MaxRP.print(dbgs(), &ST),
413  dbgs() << "RP after scheduling: ",
414  RegionMaxRP.print(dbgs(), &ST),
415  false));
416 }
417 
418 // Sort recorded regions by pressure - highest at the front
420  const auto &ST = MF.getSubtarget<GCNSubtarget>();
421  llvm::sort(Regions, [&ST, TargetOcc](const Region *R1, const Region *R2) {
422  return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
423  });
424 }
425 
426 ///////////////////////////////////////////////////////////////////////////////
427 // Legacy MaxOccupancy Strategy
428 
429 // Tries to increase occupancy applying minreg scheduler for a sequence of
430 // most demanding regions. Obtained schedules are saved as BestSchedule for a
431 // region.
432 // TargetOcc is the best achievable occupancy for a kernel.
433 // Returns better occupancy on success or current occupancy on fail.
434 // BestSchedules aren't deleted on fail.
435 unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
436  // TODO: assert Regions are sorted descending by pressure
437  const auto &ST = MF.getSubtarget<GCNSubtarget>();
438  const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
439  LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
440  << ", current = " << Occ << '\n');
441 
442  auto NewOcc = TargetOcc;
443  for (auto R : Regions) {
444  if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
445  break;
446 
447  LLVM_DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
448  printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
449 
450  BuildDAG DAG(*R, *this);
451  const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
452  const auto MaxRP = getSchedulePressure(*R, MinSchedule);
453  LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
454  printSchedRP(dbgs(), R->MaxPressure, MaxRP));
455 
456  NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST));
457  if (NewOcc <= Occ)
458  break;
459 
460  setBestSchedule(*R, MinSchedule, MaxRP);
461  }
462  LLVM_DEBUG(dbgs() << "New occupancy = " << NewOcc
463  << ", prev occupancy = " << Occ << '\n');
464  if (NewOcc > Occ) {
466  MFI->increaseOccupancy(MF, NewOcc);
467  }
468 
469  return std::max(NewOcc, Occ);
470 }
471 
473  bool TryMaximizeOccupancy) {
474  const auto &ST = MF.getSubtarget<GCNSubtarget>();
476  auto TgtOcc = MFI->getMinAllowedOccupancy();
477 
478  sortRegionsByPressure(TgtOcc);
479  auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
480 
481  if (TryMaximizeOccupancy && Occ < TgtOcc)
482  Occ = tryMaximizeOccupancy(TgtOcc);
483 
484  // This is really weird but for some magic scheduling regions twice
485  // gives performance improvement
486  const int NumPasses = Occ < TgtOcc ? 2 : 1;
487 
488  TgtOcc = std::min(Occ, TgtOcc);
489  LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
490  "target occupancy = "
491  << TgtOcc << '\n');
493  unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
494 
495  for (int I = 0; I < NumPasses; ++I) {
496  // running first pass with TargetOccupancy = 0 mimics previous scheduling
497  // approach and is a performance magic
498  LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
499  for (auto R : Regions) {
500  OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
501 
502  Ovr.schedule();
503  const auto RP = getRegionPressure(*R);
504  LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
505 
506  if (RP.getOccupancy(ST) < TgtOcc) {
507  LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
508  if (R->BestSchedule.get() &&
509  R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
510  LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
511  scheduleBest(*R);
512  } else {
513  LLVM_DEBUG(dbgs() << ", restoring\n");
514  Ovr.restoreOrder();
515  assert(R->MaxPressure.getOccupancy(ST) >= TgtOcc);
516  }
517  }
518  FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
519  }
520  }
521  MFI->limitOccupancy(FinalOccupancy);
522 }
523 
524 ///////////////////////////////////////////////////////////////////////////////
525 // Minimal Register Strategy
526 
528  const auto &ST = MF.getSubtarget<GCNSubtarget>();
530  const auto TgtOcc = MFI->getOccupancy();
531  sortRegionsByPressure(TgtOcc);
532 
533  auto MaxPressure = Regions.front()->MaxPressure;
534  for (auto R : Regions) {
535  if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
536  break;
537 
538  BuildDAG DAG(*R, *this);
539  const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
540 
541  const auto RP = getSchedulePressure(*R, MinSchedule);
542  LLVM_DEBUG(if (R->MaxPressure.less(ST, RP, TgtOcc)) {
543  dbgs() << "\nWarning: Pressure becomes worse after minreg!";
544  printSchedRP(dbgs(), R->MaxPressure, RP);
545  });
546 
547  if (!force && MaxPressure.less(ST, RP, TgtOcc))
548  break;
549 
550  scheduleRegion(*R, MinSchedule, RP);
552 
553  MaxPressure = RP;
554  }
555 }
556 
557 ///////////////////////////////////////////////////////////////////////////////
558 // ILP scheduler port
559 
561  bool TryMaximizeOccupancy) {
562  const auto &ST = MF.getSubtarget<GCNSubtarget>();
564  auto TgtOcc = MFI->getMinAllowedOccupancy();
565 
566  sortRegionsByPressure(TgtOcc);
567  auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
568 
569  if (TryMaximizeOccupancy && Occ < TgtOcc)
570  Occ = tryMaximizeOccupancy(TgtOcc);
571 
572  TgtOcc = std::min(Occ, TgtOcc);
573  LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
574  "target occupancy = "
575  << TgtOcc << '\n');
576 
577  unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
578  for (auto R : Regions) {
579  BuildDAG DAG(*R, *this);
580  const auto ILPSchedule = makeGCNILPScheduler(DAG.getBottomRoots(), *this);
581 
582  const auto RP = getSchedulePressure(*R, ILPSchedule);
583  LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
584 
585  if (RP.getOccupancy(ST) < TgtOcc) {
586  LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
587  if (R->BestSchedule.get() &&
588  R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
589  LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
590  scheduleBest(*R);
591  }
592  } else {
593  scheduleRegion(*R, ILPSchedule, RP);
595  FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
596  }
597  }
598  MFI->limitOccupancy(FinalOccupancy);
599 }
unsigned tryMaximizeOccupancy(unsigned TargetOcc=std::numeric_limits< unsigned >::max())
uint64_t CallInst * C
void enterRegion(MachineBasicBlock *BB, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned RegionInstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
ScheduleDAGTopologicalSort Topo
Topo - A topological ordering for SUnits which permits fast IsReachable and similar queries.
GCNRegPressure getRegionPressure(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End) const
LLVMContext & Context
AMDGPU specific subclass of TargetSubtarget.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:498
This is a minimal scheduler strategy.
void sortRegionsByPressure(unsigned TargetOcc)
decltype(MaxPressure) moveMaxPressure()
SpecificBumpPtrAllocator< Region > Alloc
std::vector< Region * > Regions
std::vector< MachineInstr * > detachSchedule(ScheduleRef Schedule) const
#define R2(n)
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
std::unique_ptr< MachineSchedStrategy > SchedImpl
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
void recede(SmallVectorImpl< RegisterMaskPair > *LiveUses=nullptr)
Recede across the previous instruction.
MachineFunction & MF
Machine function.
Definition: ScheduleDAG.h:560
void printRegions(raw_ostream &OS) const
Definition: BitVector.h:941
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
DbgValueVector DbgValues
Remember instruction that precedes DBG_VALUE.
void buildSchedGraph(AAResults *AA, RegPressureTracker *RPTracker=nullptr, PressureDiffs *PDiffs=nullptr, LiveIntervals *LIS=nullptr, bool TrackLaneMasks=false)
Builds SUnits for the current region.
void InitDAGTopologicalSorting()
Creates the initial topological ordering from the DAG to be scheduled.
void scheduleLegacyMaxOccupancy(bool TryMaximizeOccupancy=true)
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
Definition: SlotIndexes.h:254
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
static LLVM_DUMP_METHOD void printLivenessInfo(raw_ostream &OS, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, const LiveIntervals *LIS)
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringLiteral > StandardNames)
Initialize the set of available library functions based on the specified target triple.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
static MachineInstr * getMachineInstr(MachineInstr *MI)
List of registers defined and used by a machine instruction.
void printSchedRP(raw_ostream &OS, const GCNRegPressure &Before, const GCNRegPressure &After) const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
#define P(N)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
void setBestSchedule(Region &R, ScheduleRef Schedule, const GCNRegPressure &MaxRP=GCNRegPressure())
unsigned const MachineRegisterInfo * MRI
const MachineInstr * getLastTrackedMI() const
std::vector< const SUnit * > makeGCNILPScheduler(ArrayRef< const SUnit * > BotRoots, const ScheduleDAG &DAG)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:156
std::vector< const SUnit * > makeMinRegSchedule(ArrayRef< const SUnit * > TopRoots, const ScheduleDAG &DAG)
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:339
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void findRootsAndBiasEdges(SmallVectorImpl< SUnit * > &TopRoots, SmallVectorImpl< SUnit * > &BotRoots)
This file defines the class GCNIterativeScheduler, which uses an iterative approach to find a best sc...
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1439
const MachineFrameInfo & MFI
GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, const LiveIntervals &LIS)
void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the Regi...
GCNRegPressure getSchedulePressure(const Region &R, Range &&Schedule) const
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:350
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1116
OverrideLegacyStrategy(Region &R, MachineSchedStrategy &OverrideStrategy, GCNIterativeScheduler &_Sch)
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
const MachineBasicBlock::iterator End
static LLVM_DUMP_METHOD void printRegion(raw_ostream &OS, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, const LiveIntervals *LIS, unsigned MaxInstNum=std::numeric_limits< unsigned >::max())
ArrayRef< const SUnit * > getTopRoots() const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
Representation of each machine instruction.
Definition: MachineInstr.h:62
LiveIntervals * LIS
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
const TargetRegisterInfo * TRI
Target processor register info.
Definition: ScheduleDAG.h:559
void printSchedResult(raw_ostream &OS, const Region *R, const GCNRegPressure &RP) const
GCNIterativeScheduler(MachineSchedContext *C, StrategyKind S)
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
MachineSchedStrategy - Interface to the scheduling algorithm used by ScheduleDAGMI.
#define I(x, y, z)
Definition: MD5.cpp:59
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
void scheduleRegion(Region &R, Range &&Schedule, const GCNRegPressure &MaxRP=GCNRegPressure())
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void print(raw_ostream &OS, const GCNSubtarget *ST=nullptr) const
void scheduleMinReg(bool force=false)
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
MachineBasicBlock * BB
The block in which to insert instructions.
IRTranslator LLVM IR MI
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1556
void scheduleILP(bool TryMaximizeOccupancy=true)
MachineRegisterInfo & MRI
Virtual/real register map.
Definition: ScheduleDAG.h:561
std::vector< SUnit > SUnits
The scheduling units.
Definition: ScheduleDAG.h:562
#define LLVM_DEBUG(X)
Definition: Debug.h:122
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1525
void recede(const MachineInstr &MI)
RegPressureTracker RPTracker
void handleMove(MachineInstr &MI, bool UpdateFlags=false)
Call this method to notify LiveIntervals that instruction MI has been moved within a basic block.
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242