LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - GCNIterativeScheduler.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 175 192 91.1 %
Date: 2018-02-23 15:42:53 Functions: 25 31 80.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- GCNIterativeScheduler.cpp ------------------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : 
      10             : #include "GCNIterativeScheduler.h"
      11             : #include "AMDGPUSubtarget.h"
      12             : #include "GCNRegPressure.h"
      13             : #include "GCNSchedStrategy.h"
      14             : #include "llvm/ADT/ArrayRef.h"
      15             : #include "llvm/ADT/STLExtras.h"
      16             : #include "llvm/ADT/SmallVector.h"
      17             : #include "llvm/CodeGen/LiveIntervals.h"
      18             : #include "llvm/CodeGen/MachineBasicBlock.h"
      19             : #include "llvm/CodeGen/MachineFunction.h"
      20             : #include "llvm/CodeGen/RegisterPressure.h"
      21             : #include "llvm/CodeGen/ScheduleDAG.h"
      22             : #include "llvm/Support/Compiler.h"
      23             : #include "llvm/Support/Debug.h"
      24             : #include "llvm/Support/raw_ostream.h"
      25             : #include <algorithm>
      26             : #include <cassert>
      27             : #include <iterator>
      28             : #include <limits>
      29             : #include <memory>
      30             : #include <type_traits>
      31             : #include <vector>
      32             : 
      33             : using namespace llvm;
      34             : 
      35             : #define DEBUG_TYPE "machine-scheduler"
      36             : 
      37             : namespace llvm {
      38             : 
      39             : std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
      40             :                                               const ScheduleDAG &DAG);
      41             : 
      42             :   std::vector<const SUnit*> makeGCNILPScheduler(ArrayRef<const SUnit*> BotRoots,
      43             :     const ScheduleDAG &DAG);
      44             : }
      45             : 
      46             : // shim accessors for different order containers
      47             : static inline MachineInstr *getMachineInstr(MachineInstr *MI) {
      48             :   return MI;
      49             : }
      50             : static inline MachineInstr *getMachineInstr(const SUnit *SU) {
      51        3413 :   return SU->getInstr();
      52             : }
      53             : static inline MachineInstr *getMachineInstr(const SUnit &SU) {
      54         228 :   return SU.getInstr();
      55             : }
      56             : 
      57             : #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
      58             : LLVM_DUMP_METHOD
      59             : static void printRegion(raw_ostream &OS,
      60             :                         MachineBasicBlock::iterator Begin,
      61             :                         MachineBasicBlock::iterator End,
      62             :                         const LiveIntervals *LIS,
      63             :                         unsigned MaxInstNum =
      64             :                           std::numeric_limits<unsigned>::max()) {
      65             :   auto BB = Begin->getParent();
      66             :   OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
      67             :      << BB->getName() << ":\n";
      68             :   auto I = Begin;
      69             :   MaxInstNum = std::max(MaxInstNum, 1u);
      70             :   for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
      71             :     if (!I->isDebugValue() && LIS)
      72             :       OS << LIS->getInstructionIndex(*I);
      73             :     OS << '\t' << *I;
      74             :   }
      75             :   if (I != End) {
      76             :     OS << "\t...\n";
      77             :     I = std::prev(End);
      78             :     if (!I->isDebugValue() && LIS)
      79             :       OS << LIS->getInstructionIndex(*I);
      80             :     OS << '\t' << *I;
      81             :   }
      82             :   if (End != BB->end()) { // print boundary inst if present
      83             :     OS << "----\n";
      84             :     if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
      85             :     OS << *End;
      86             :   }
      87             : }
      88             : 
      89             : LLVM_DUMP_METHOD
      90             : static void printLivenessInfo(raw_ostream &OS,
      91             :                               MachineBasicBlock::iterator Begin,
      92             :                               MachineBasicBlock::iterator End,
      93             :                               const LiveIntervals *LIS) {
      94             :   const auto BB = Begin->getParent();
      95             :   const auto &MRI = BB->getParent()->getRegInfo();
      96             : 
      97             :   const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
      98             :   OS << "LIn RP: ";
      99             :   getRegPressure(MRI, LiveIns).print(OS);
     100             : 
     101             :   const auto BottomMI = End == BB->end() ? std::prev(End) : End;
     102             :   const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
     103             :   OS << "LOt RP: ";
     104             :   getRegPressure(MRI, LiveOuts).print(OS);
     105             : }
     106             : 
     107             : LLVM_DUMP_METHOD
     108             : void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
     109             :   const auto &ST = MF.getSubtarget<SISubtarget>();
     110             :   for (const auto R : Regions) {
     111             :     OS << "Region to schedule ";
     112             :     printRegion(OS, R->Begin, R->End, LIS, 1);
     113             :     printLivenessInfo(OS, R->Begin, R->End, LIS);
     114             :     OS << "Max RP: ";
     115             :     R->MaxPressure.print(OS, &ST);
     116             :   }
     117             : }
     118             : 
     119             : LLVM_DUMP_METHOD
     120             : void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
     121             :                                              const Region *R,
     122             :                                              const GCNRegPressure &RP) const {
     123             :   OS << "\nAfter scheduling ";
     124             :   printRegion(OS, R->Begin, R->End, LIS);
     125             :   printSchedRP(OS, R->MaxPressure, RP);
     126             :   OS << '\n';
     127             : }
     128             : 
     129             : LLVM_DUMP_METHOD
     130             : void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
     131             :                                          const GCNRegPressure &Before,
     132             :                                          const GCNRegPressure &After) const {
     133             :   const auto &ST = MF.getSubtarget<SISubtarget>();
     134             :   OS << "RP before: ";
     135             :   Before.print(OS, &ST);
     136             :   OS << "RP after:  ";
     137             :   After.print(OS, &ST);
     138             : }
     139             : #endif
     140             : 
     141             : // DAG builder helper
     142             : class GCNIterativeScheduler::BuildDAG {
     143             :   GCNIterativeScheduler &Sch;
     144             :   SmallVector<SUnit *, 8> TopRoots;
     145             : 
     146             :   SmallVector<SUnit*, 8> BotRoots;
     147             : public:
     148           7 :   BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
     149           7 :     : Sch(_Sch) {
     150           7 :     auto BB = R.Begin->getParent();
     151           7 :     Sch.BaseClass::startBlock(BB);
     152           7 :     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
     153             : 
     154           7 :     Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
     155             :                         /*TrackLaneMask*/true);
     156           7 :     Sch.Topo.InitDAGTopologicalSorting();
     157           7 :     Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
     158           7 :   }
     159             : 
     160          14 :   ~BuildDAG() {
     161           7 :     Sch.BaseClass::exitRegion();
     162           7 :     Sch.BaseClass::finishBlock();
     163           7 :   }
     164             : 
     165             :   ArrayRef<const SUnit *> getTopRoots() const {
     166             :     return TopRoots;
     167             :   }
     168             :   ArrayRef<SUnit*> getBottomRoots() const {
     169             :     return BotRoots;
     170             :   }
     171             : };
     172             : 
     173             : class GCNIterativeScheduler::OverrideLegacyStrategy {
     174             :   GCNIterativeScheduler &Sch;
     175             :   Region &Rgn;
     176             :   std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
     177             :   GCNRegPressure SaveMaxRP;
     178             : 
     179             : public:
     180           5 :   OverrideLegacyStrategy(Region &R,
     181             :                          MachineSchedStrategy &OverrideStrategy,
     182             :                          GCNIterativeScheduler &_Sch)
     183           5 :     : Sch(_Sch)
     184             :     , Rgn(R)
     185             :     , SaveSchedImpl(std::move(_Sch.SchedImpl))
     186          10 :     , SaveMaxRP(R.MaxPressure) {
     187             :     Sch.SchedImpl.reset(&OverrideStrategy);
     188           5 :     auto BB = R.Begin->getParent();
     189           5 :     Sch.BaseClass::startBlock(BB);
     190           5 :     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
     191           5 :   }
     192             : 
     193          10 :   ~OverrideLegacyStrategy() {
     194           5 :     Sch.BaseClass::exitRegion();
     195           5 :     Sch.BaseClass::finishBlock();
     196           5 :     Sch.SchedImpl.release();
     197             :     Sch.SchedImpl = std::move(SaveSchedImpl);
     198           5 :   }
     199             : 
     200           5 :   void schedule() {
     201             :     assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
     202             :     DEBUG(dbgs() << "\nScheduling ";
     203             :       printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
     204           5 :     Sch.BaseClass::schedule();
     205             : 
     206             :     // Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
     207           5 :     Sch.RegionEnd = Rgn.End;
     208             :     //assert(Rgn.End == Sch.RegionEnd);
     209           5 :     Rgn.Begin = Sch.RegionBegin;
     210             :     Rgn.MaxPressure.clear();
     211           5 :   }
     212             : 
     213             :   void restoreOrder() {
     214             :     assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
     215             :     // DAG SUnits are stored using original region's order
     216             :     // so just use SUnits as the restoring schedule
     217           1 :     Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
     218             :   }
     219             : };
     220             : 
     221             : namespace {
     222             : 
     223             : // just a stub to make base class happy
     224          14 : class SchedStrategyStub : public MachineSchedStrategy {
     225             : public:
     226          14 :   bool shouldTrackPressure() const override { return false; }
     227          14 :   bool shouldTrackLaneMasks() const override { return false; }
     228           0 :   void initialize(ScheduleDAGMI *DAG) override {}
     229           0 :   SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
     230           0 :   void schedNode(SUnit *SU, bool IsTopNode) override {}
     231           0 :   void releaseTopNode(SUnit *SU) override {}
     232           0 :   void releaseBottomNode(SUnit *SU) override {}
     233             : };
     234             : 
     235             : } // end anonymous namespace
     236             : 
     237           7 : GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
     238           7 :                                              StrategyKind S)
     239             :   : BaseClass(C, llvm::make_unique<SchedStrategyStub>())
     240             :   , Context(C)
     241             :   , Strategy(S)
     242          28 :   , UPTracker(*LIS) {
     243           7 : }
     244             : 
     245             : // returns max pressure for a region
     246             : GCNRegPressure
     247          12 : GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
     248             :                                          MachineBasicBlock::iterator End)
     249             :   const {
     250             :   // For the purpose of pressure tracking bottom inst of the region should
     251             :   // be also processed. End is either BB end, BB terminator inst or sched
     252             :   // boundary inst.
     253          12 :   auto const BBEnd = Begin->getParent()->end();
     254          12 :   auto const BottomMI = End == BBEnd ? std::prev(End) : End;
     255             : 
     256             :   // scheduleRegions walks bottom to top, so its likely we just get next
     257             :   // instruction to track
     258             :   auto AfterBottomMI = std::next(BottomMI);
     259          12 :   if (AfterBottomMI == BBEnd ||
     260           0 :       &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
     261          12 :     UPTracker.reset(*BottomMI);
     262             :   } else {
     263             :     assert(UPTracker.isValid());
     264             :   }
     265             : 
     266        3855 :   for (auto I = BottomMI; I != Begin; --I)
     267        3831 :     UPTracker.recede(*I);
     268             : 
     269          12 :   UPTracker.recede(*Begin);
     270             : 
     271             :   assert(UPTracker.isValid() ||
     272             :          (dbgs() << "Tracked region ",
     273             :           printRegion(dbgs(), Begin, End, LIS), false));
     274          12 :   return UPTracker.moveMaxPressure();
     275             : }
     276             : 
     277             : // returns max pressure for a tentative schedule
     278             : template <typename Range> GCNRegPressure
     279           7 : GCNIterativeScheduler::getSchedulePressure(const Region &R,
     280             :                                            Range &&Schedule) const {
     281           7 :   auto const BBEnd = R.Begin->getParent()->end();
     282           7 :   GCNUpwardRPTracker RPTracker(*LIS);
     283           7 :   if (R.End != BBEnd) {
     284             :     // R.End points to the boundary instruction but the
     285             :     // schedule doesn't include it
     286           7 :     RPTracker.reset(*R.End);
     287           7 :     RPTracker.recede(*R.End);
     288             :   } else {
     289             :     // R.End doesn't point to the boundary instruction
     290           0 :     RPTracker.reset(*std::prev(BBEnd));
     291             :   }
     292        2204 :   for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
     293        4394 :     RPTracker.recede(*getMachineInstr(*--I));
     294             :   }
     295           7 :   return RPTracker.moveMaxPressure();
     296             : }
     297             : 
     298           7 : void GCNIterativeScheduler::enterRegion(MachineBasicBlock *BB, // overriden
     299             :                                         MachineBasicBlock::iterator Begin,
     300             :                                         MachineBasicBlock::iterator End,
     301             :                                         unsigned NumRegionInstrs) {
     302           7 :   BaseClass::enterRegion(BB, Begin, End, NumRegionInstrs);
     303           7 :   if (NumRegionInstrs > 2) {
     304           7 :     Regions.push_back(
     305          21 :       new (Alloc.Allocate())
     306             :       Region { Begin, End, NumRegionInstrs,
     307           7 :                getRegionPressure(Begin, End), nullptr });
     308             :   }
     309           7 : }
     310             : 
     311           7 : void GCNIterativeScheduler::schedule() { // overriden
     312             :   // do nothing
     313             :   DEBUG(
     314             :     printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
     315             :     if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
     316             :       dbgs() << "Max RP: ";
     317             :       Regions.back()->MaxPressure.print(dbgs(), &MF.getSubtarget<SISubtarget>());
     318             :     }
     319             :     dbgs() << '\n';
     320             :   );
     321           7 : }
     322             : 
     323           7 : void GCNIterativeScheduler::finalizeSchedule() { // overriden
     324           7 :   if (Regions.empty())
     325             :     return;
     326           7 :   switch (Strategy) {
     327           0 :   case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
     328           3 :   case SCHEDULE_MINREGFORCED: scheduleMinReg(true); break;
     329           3 :   case SCHEDULE_LEGACYMAXOCCUPANCY: scheduleLegacyMaxOccupancy(); break;
     330           1 :   case SCHEDULE_ILP: scheduleILP(false); break;
     331             :   }
     332             : }
     333             : 
     334             : // Detach schedule from SUnits and interleave it with debug values.
     335             : // Returned schedule becomes independent of DAG state.
     336             : std::vector<MachineInstr*>
     337           2 : GCNIterativeScheduler::detachSchedule(ScheduleRef Schedule) const {
     338             :   std::vector<MachineInstr*> Res;
     339           2 :   Res.reserve(Schedule.size() * 2);
     340             : 
     341           2 :   if (FirstDbgValue)
     342           0 :     Res.push_back(FirstDbgValue);
     343             : 
     344           2 :   const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
     345        1518 :   for (auto SU : Schedule) {
     346        1516 :     Res.push_back(SU->getInstr());
     347             :     const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
     348           0 :       return P.second == SU->getInstr();
     349             :     });
     350         758 :     if (D != DbgE)
     351           0 :       Res.push_back(D->first);
     352             :   }
     353           2 :   return Res;
     354             : }
     355             : 
     356           2 : void GCNIterativeScheduler::setBestSchedule(Region &R,
     357             :                                             ScheduleRef Schedule,
     358             :                                             const GCNRegPressure &MaxRP) {
     359           6 :   R.BestSchedule.reset(
     360           2 :     new TentativeSchedule{ detachSchedule(Schedule), MaxRP });
     361           2 : }
     362             : 
     363           2 : void GCNIterativeScheduler::scheduleBest(Region &R) {
     364             :   assert(R.BestSchedule.get() && "No schedule specified");
     365           2 :   scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
     366             :   R.BestSchedule.reset();
     367           2 : }
     368             : 
     369             : // minimal required region scheduler, works for ranges of SUnits*,
     370             : // SUnits or MachineIntrs*
     371             : template <typename Range>
     372           7 : void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
     373             :                                            const GCNRegPressure &MaxRP) {
     374             :   assert(RegionBegin == R.Begin && RegionEnd == R.End);
     375             :   assert(LIS != nullptr);
     376             : #ifndef NDEBUG
     377             :   const auto SchedMaxRP = getSchedulePressure(R, Schedule);
     378             : #endif
     379           7 :   auto BB = R.Begin->getParent();
     380             :   auto Top = R.Begin;
     381         992 :   for (const auto &I : Schedule) {
     382        1970 :     auto MI = getMachineInstr(I);
     383        2197 :     if (MI != &*Top) {
     384             :       BB->remove(MI);
     385             :       BB->insert(Top, MI);
     386        1656 :       if (!MI->isDebugValue())
     387        1656 :         LIS->handleMove(*MI, true);
     388             :     }
     389        2197 :     if (!MI->isDebugValue()) {
     390             :       // Reset read - undef flags and update them later.
     391       23933 :       for (auto &Op : MI->operands())
     392       18428 :         if (Op.isReg() && Op.isDef())
     393             :           Op.setIsUndef(false);
     394             : 
     395        2197 :       RegisterOperands RegOpers;
     396        2197 :       RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
     397             :                                        /*IgnoreDead*/false);
     398             :       // Adjust liveness and add missing dead+read-undef flags.
     399        4394 :       auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
     400        2197 :       RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
     401             :     }
     402        2197 :     Top = std::next(MI->getIterator());
     403             :   }
     404          11 :   RegionBegin = getMachineInstr(Schedule.front());
     405             : 
     406             :   // Schedule consisting of MachineInstr* is considered 'detached'
     407             :   // and already interleaved with debug values
     408             :   if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
     409           7 :     placeDebugValues();
     410             :     // Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
     411             :     //assert(R.End == RegionEnd);
     412           7 :     RegionEnd = R.End;
     413             :   }
     414             : 
     415           7 :   R.Begin = RegionBegin;
     416           7 :   R.MaxPressure = MaxRP;
     417             : 
     418             : #ifndef NDEBUG
     419             :   const auto RegionMaxRP = getRegionPressure(R);
     420             :   const auto &ST = MF.getSubtarget<SISubtarget>();
     421             : #endif
     422             :   assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP))
     423             :   || (dbgs() << "Max RP mismatch!!!\n"
     424             :                 "RP for schedule (calculated): ",
     425             :       SchedMaxRP.print(dbgs(), &ST),
     426             :       dbgs() << "RP for schedule (reported): ",
     427             :       MaxRP.print(dbgs(), &ST),
     428             :       dbgs() << "RP after scheduling: ",
     429             :       RegionMaxRP.print(dbgs(), &ST),
     430             :       false));
     431           7 : }
     432             : 
     433             : // Sort recorded regions by pressure - highest at the front
     434           7 : void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
     435           7 :   const auto &ST = MF.getSubtarget<SISubtarget>();
     436             :   std::sort(Regions.begin(), Regions.end(),
     437             :     [&ST, TargetOcc](const Region *R1, const Region *R2) {
     438           0 :     return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
     439           0 :   });
     440           7 : }
     441             : 
     442             : ///////////////////////////////////////////////////////////////////////////////
     443             : // Legacy MaxOccupancy Strategy
     444             : 
     445             : // Tries to increase occupancy applying minreg scheduler for a sequence of
     446             : // most demanding regions. Obtained schedules are saved as BestSchedule for a
     447             : // region.
     448             : // TargetOcc is the best achievable occupancy for a kernel.
     449             : // Returns better occupancy on success or current occupancy on fail.
     450             : // BestSchedules aren't deleted on fail.
     451           3 : unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
     452             :   // TODO: assert Regions are sorted descending by pressure
     453           3 :   const auto &ST = MF.getSubtarget<SISubtarget>();
     454           3 :   const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
     455             :   DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
     456             :                << ", current = " << Occ << '\n');
     457             : 
     458           3 :   auto NewOcc = TargetOcc;
     459           5 :   for (auto R : Regions) {
     460           3 :     if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
     461             :       break;
     462             : 
     463             :     DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
     464             :           printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
     465             : 
     466           5 :     BuildDAG DAG(*R, *this);
     467           6 :     const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
     468           3 :     const auto MaxRP = getSchedulePressure(*R, MinSchedule);
     469             :     DEBUG(dbgs() << "Occupancy improvement attempt:\n";
     470             :           printSchedRP(dbgs(), R->MaxPressure, MaxRP));
     471             : 
     472           6 :     NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST));
     473           3 :     if (NewOcc <= Occ)
     474             :       break;
     475             : 
     476           2 :     setBestSchedule(*R, MinSchedule, MaxRP);
     477             :   }
     478             :   DEBUG(dbgs() << "New occupancy = " << NewOcc
     479             :                << ", prev occupancy = " << Occ << '\n');
     480           3 :   return std::max(NewOcc, Occ);
     481             : }
     482             : 
     483           3 : void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
     484             :   bool TryMaximizeOccupancy) {
     485           3 :   const auto &ST = MF.getSubtarget<SISubtarget>();
     486           3 :   auto TgtOcc = ST.getOccupancyWithLocalMemSize(MF);
     487             : 
     488           3 :   sortRegionsByPressure(TgtOcc);
     489           3 :   auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
     490             : 
     491           3 :   if (TryMaximizeOccupancy && Occ < TgtOcc)
     492           3 :     Occ = tryMaximizeOccupancy(TgtOcc);
     493             : 
     494             :   // This is really weird but for some magic scheduling regions twice
     495             :   // gives performance improvement
     496           3 :   const int NumPasses = Occ < TgtOcc ? 2 : 1;
     497             : 
     498           3 :   TgtOcc = std::min(Occ, TgtOcc);
     499             :   DEBUG(dbgs() << "Scheduling using default scheduler, "
     500             :                   "target occupancy = " << TgtOcc << '\n');
     501           3 :   GCNMaxOccupancySchedStrategy LStrgy(Context);
     502             : 
     503          13 :   for (int I = 0; I < NumPasses; ++I) {
     504             :     // running first pass with TargetOccupancy = 0 mimics previous scheduling
     505             :     // approach and is a performance magic
     506           5 :     LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
     507          10 :     for (auto R : Regions) {
     508          10 :       OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
     509             : 
     510           5 :       Ovr.schedule();
     511           5 :       const auto RP = getRegionPressure(*R);
     512             :       DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
     513             : 
     514           5 :       if (RP.getOccupancy(ST) < TgtOcc) {
     515             :         DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
     516           5 :         if (R->BestSchedule.get() &&
     517           2 :             R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
     518             :           DEBUG(dbgs() << ", scheduling minimal register\n");
     519           2 :           scheduleBest(*R);
     520             :         } else {
     521             :           DEBUG(dbgs() << ", restoring\n");
     522             :           Ovr.restoreOrder();
     523             :           assert(R->MaxPressure.getOccupancy(ST) >= TgtOcc);
     524             :         }
     525             :       }
     526             :     }
     527             :   }
     528           3 : }
     529             : 
     530             : ///////////////////////////////////////////////////////////////////////////////
     531             : // Minimal Register Strategy
     532             : 
     533           3 : void GCNIterativeScheduler::scheduleMinReg(bool force) {
     534           3 :   const auto &ST = MF.getSubtarget<SISubtarget>();
     535           3 :   const auto TgtOcc = ST.getOccupancyWithLocalMemSize(MF);
     536           3 :   sortRegionsByPressure(TgtOcc);
     537             : 
     538           3 :   auto MaxPressure = Regions.front()->MaxPressure;
     539           6 :   for (auto R : Regions) {
     540           3 :     if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
     541             :       break;
     542             : 
     543           6 :     BuildDAG DAG(*R, *this);
     544           6 :     const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
     545             : 
     546           3 :     const auto RP = getSchedulePressure(*R, MinSchedule);
     547             :     DEBUG(if (R->MaxPressure.less(ST, RP, TgtOcc)) {
     548             :       dbgs() << "\nWarning: Pressure becomes worse after minreg!";
     549             :       printSchedRP(dbgs(), R->MaxPressure, RP);
     550             :     });
     551             : 
     552           3 :     if (!force && MaxPressure.less(ST, RP, TgtOcc))
     553             :       break;
     554             : 
     555           3 :     scheduleRegion(*R, MinSchedule, RP);
     556             :     DEBUG(printSchedResult(dbgs(), R, RP));
     557             : 
     558           3 :     MaxPressure = RP;
     559             :   }
     560           3 : }
     561             : 
     562             : ///////////////////////////////////////////////////////////////////////////////
     563             : // ILP scheduler port
     564             : 
     565           1 : void GCNIterativeScheduler::scheduleILP(
     566             :   bool TryMaximizeOccupancy) {
     567           1 :   const auto &ST = MF.getSubtarget<SISubtarget>();
     568           3 :   auto TgtOcc = std::min(ST.getOccupancyWithLocalMemSize(MF),
     569           3 :                          ST.getWavesPerEU(MF.getFunction()).second);
     570             : 
     571           1 :   sortRegionsByPressure(TgtOcc);
     572           1 :   auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
     573             : 
     574           1 :   if (TryMaximizeOccupancy && Occ < TgtOcc)
     575           0 :     Occ = tryMaximizeOccupancy(TgtOcc);
     576             : 
     577           1 :   TgtOcc = std::min(Occ, TgtOcc);
     578             :   DEBUG(dbgs() << "Scheduling using default scheduler, "
     579             :     "target occupancy = " << TgtOcc << '\n');
     580             : 
     581           2 :   for (auto R : Regions) {
     582           2 :     BuildDAG DAG(*R, *this);
     583           2 :     const auto ILPSchedule = makeGCNILPScheduler(DAG.getBottomRoots(), *this);
     584             : 
     585           1 :     const auto RP = getSchedulePressure(*R, ILPSchedule);
     586             :     DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
     587             : 
     588           1 :     if (RP.getOccupancy(ST) < TgtOcc) {
     589             :       DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
     590           0 :       if (R->BestSchedule.get() &&
     591           0 :         R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
     592             :         DEBUG(dbgs() << ", scheduling minimal register\n");
     593           0 :         scheduleBest(*R);
     594             :       }
     595             :     } else {
     596           1 :       scheduleRegion(*R, ILPSchedule, RP);
     597             :       DEBUG(printSchedResult(dbgs(), R, RP));
     598             :     }
     599             :   }
     600           1 : }

Generated by: LCOV version 1.13