LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - GCNSchedStrategy.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 213 231 92.2 %
Date: 2018-10-20 13:21:21 Functions: 11 11 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This contains a MachineSchedStrategy implementation for maximizing wave
      12             : /// occupancy on GCN hardware.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "GCNSchedStrategy.h"
      16             : #include "AMDGPUSubtarget.h"
      17             : #include "SIInstrInfo.h"
      18             : #include "SIMachineFunctionInfo.h"
      19             : #include "SIRegisterInfo.h"
      20             : #include "llvm/CodeGen/RegisterClassInfo.h"
      21             : #include "llvm/Support/MathExtras.h"
      22             : 
      23             : #define DEBUG_TYPE "machine-scheduler"
      24             : 
      25             : using namespace llvm;
      26             : 
      27       19518 : GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
      28       19518 :     const MachineSchedContext *C) :
      29       19518 :     GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
      30             : 
      31       21452 : void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
      32       21452 :   GenericScheduler::initialize(DAG);
      33             : 
      34       21452 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
      35             : 
      36       21452 :   MF = &DAG->MF;
      37             : 
      38       21452 :   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
      39             : 
      40             :   // FIXME: This is also necessary, because some passes that run after
      41             :   // scheduling and before regalloc increase register pressure.
      42             :   const int ErrorMargin = 3;
      43             : 
      44       42904 :   SGPRExcessLimit = Context->RegClassInfo
      45       21452 :     ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
      46       42904 :   VGPRExcessLimit = Context->RegClassInfo
      47       21452 :     ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
      48       21452 :   if (TargetOccupancy) {
      49         341 :     SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
      50         341 :     VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
      51             :   } else {
      52       21111 :     SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      53             :                                                     SRI->getSGPRPressureSet());
      54       21111 :     VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      55             :                                                     SRI->getVGPRPressureSet());
      56             :   }
      57             : 
      58       21452 :   SGPRCriticalLimit -= ErrorMargin;
      59       21452 :   VGPRCriticalLimit -= ErrorMargin;
      60       21452 : }
      61             : 
      62     3323073 : void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
      63             :                                      bool AtTop, const RegPressureTracker &RPTracker,
      64             :                                      const SIRegisterInfo *SRI,
      65             :                                      unsigned SGPRPressure,
      66             :                                      unsigned VGPRPressure) {
      67             : 
      68     3323073 :   Cand.SU = SU;
      69     3323073 :   Cand.AtTop = AtTop;
      70             : 
      71             :   // getDownwardPressure() and getUpwardPressure() make temporary changes to
      72             :   // the tracker, so we need to pass those function a non-const copy.
      73             :   RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
      74             : 
      75             :   std::vector<unsigned> Pressure;
      76             :   std::vector<unsigned> MaxPressure;
      77             : 
      78     3323073 :   if (AtTop)
      79      569742 :     TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
      80             :   else {
      81             :     // FIXME: I think for bottom up scheduling, the register pressure is cached
      82             :     // and can be retrieved by DAG->getPressureDif(SU).
      83     2753331 :     TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
      84             :   }
      85             : 
      86     3323073 :   unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
      87     3323073 :   unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
      88             : 
      89             :   // If two instructions increase the pressure of different register sets
      90             :   // by the same amount, the generic scheduler will prefer to schedule the
      91             :   // instruction that increases the set with the least amount of registers,
      92             :   // which in our case would be SGPRs.  This is rarely what we want, so
      93             :   // when we report excess/critical register pressure, we do it either
      94             :   // only for VGPRs or only for SGPRs.
      95             : 
      96             :   // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
      97             :   const unsigned MaxVGPRPressureInc = 16;
      98     3323073 :   bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
      99     3323073 :   bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
     100             : 
     101             : 
     102             :   // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
     103             :   // to increase the likelihood we don't go over the limits.  We should improve
     104             :   // the analysis to look through dependencies to find the path with the least
     105             :   // register pressure.
     106             : 
     107             :   // We only need to update the RPDelata for instructions that increase
     108             :   // register pressure.  Instructions that decrease or keep reg pressure
     109             :   // the same will be marked as RegExcess in tryCandidate() when they
     110             :   // are compared with instructions that increase the register pressure.
     111     3323073 :   if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
     112       10886 :     Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
     113       10886 :     Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
     114             :   }
     115             : 
     116     3323073 :   if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
     117       25416 :     Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
     118       25416 :     Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
     119             :   }
     120             : 
     121             :   // Register pressure is considered 'CRITICAL' if it is approaching a value
     122             :   // that would reduce the wave occupancy for the execution unit.  When
     123             :   // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
     124             :   // has the same cost, so we don't need to prefer one over the other.
     125             : 
     126     3323073 :   int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
     127     3323073 :   int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
     128             : 
     129     3323073 :   if (SGPRDelta >= 0 || VGPRDelta >= 0) {
     130     1130348 :     if (SGPRDelta > VGPRDelta) {
     131      260248 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
     132             :       Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
     133             :     } else {
     134     2000448 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
     135             :       Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
     136             :     }
     137             :   }
     138     3323073 : }
     139             : 
     140             : // This function is mostly cut and pasted from
     141             : // GenericScheduler::pickNodeFromQueue()
     142      314616 : void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
     143             :                                          const CandPolicy &ZonePolicy,
     144             :                                          const RegPressureTracker &RPTracker,
     145             :                                          SchedCandidate &Cand) {
     146      314616 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
     147             :   ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
     148      314616 :   unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
     149      629232 :   unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
     150             :   ReadyQueue &Q = Zone.Available;
     151     3637689 :   for (SUnit *SU : Q) {
     152             : 
     153             :     SchedCandidate TryCand(ZonePolicy);
     154     3323073 :     initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
     155             :                   SGPRPressure, VGPRPressure);
     156             :     // Pass SchedBoundary only when comparing nodes from the same boundary.
     157     3323073 :     SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
     158     3323073 :     GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
     159     3323073 :     if (TryCand.Reason != NoCand) {
     160             :       // Initialize resource delta if needed in case future heuristics query it.
     161     1164983 :       if (TryCand.ResDelta == SchedResourceDelta())
     162     1164983 :         TryCand.initResourceDelta(Zone.DAG, SchedModel);
     163             :       Cand.setBest(TryCand);
     164             :     }
     165             :   }
     166      314616 : }
     167             : 
     168             : // This function is mostly cut and pasted from
     169             : // GenericScheduler::pickNodeBidirectional()
     170      341436 : SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
     171             :   // Schedule as far as possible in the direction of no choice. This is most
     172             :   // efficient, but also provides the best heuristics for CriticalPSets.
     173      341436 :   if (SUnit *SU = Bot.pickOnlyChoice()) {
     174       64167 :     IsTopNode = false;
     175       64167 :     return SU;
     176             :   }
     177      277269 :   if (SUnit *SU = Top.pickOnlyChoice()) {
     178        6902 :     IsTopNode = true;
     179        6902 :     return SU;
     180             :   }
     181             :   // Set the bottom-up policy based on the state of the current bottom zone and
     182             :   // the instructions outside the zone, including the top zone.
     183      270367 :   CandPolicy BotPolicy;
     184      270367 :   setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
     185             :   // Set the top-down policy based on the state of the current top zone and
     186             :   // the instructions outside the zone, including the bottom zone.
     187      270367 :   CandPolicy TopPolicy;
     188      270367 :   setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
     189             : 
     190             :   // See if BotCand is still valid (because we previously scheduled from Top).
     191             :   LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
     192      270367 :   if (!BotCand.isValid() || BotCand.SU->isScheduled ||
     193             :       BotCand.Policy != BotPolicy) {
     194      218227 :     BotCand.reset(CandPolicy());
     195      436454 :     pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
     196             :     assert(BotCand.Reason != NoCand && "failed to find the first candidate");
     197             :   } else {
     198             :     LLVM_DEBUG(traceCandidate(BotCand));
     199             :   }
     200             : 
     201             :   // Check if the top Q has a better candidate.
     202             :   LLVM_DEBUG(dbgs() << "Picking from Top:\n");
     203      270367 :   if (!TopCand.isValid() || TopCand.SU->isScheduled ||
     204             :       TopCand.Policy != TopPolicy) {
     205       96389 :     TopCand.reset(CandPolicy());
     206      192778 :     pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
     207             :     assert(TopCand.Reason != NoCand && "failed to find the first candidate");
     208             :   } else {
     209             :     LLVM_DEBUG(traceCandidate(TopCand));
     210             :   }
     211             : 
     212             :   // Pick best from BotCand and TopCand.
     213             :   LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
     214             :              dbgs() << "Bot Cand: "; traceCandidate(BotCand););
     215             :   SchedCandidate Cand;
     216      270367 :   if (TopCand.Reason == BotCand.Reason) {
     217      122747 :     Cand = BotCand;
     218             :     GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
     219      122747 :     TopCand.Reason = NoCand;
     220      122747 :     GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
     221      122747 :     if (TopCand.Reason != NoCand) {
     222             :       Cand.setBest(TopCand);
     223             :     } else {
     224      104468 :       TopCand.Reason = TopReason;
     225             :     }
     226             :   } else {
     227      147620 :     if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
     228           9 :       Cand = TopCand;
     229      147611 :     } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
     230         130 :       Cand = BotCand;
     231      147481 :     } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     232         402 :       Cand = TopCand;
     233      147079 :     } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     234       12151 :       Cand = BotCand;
     235             :     } else {
     236      134928 :       if (BotCand.Reason > TopCand.Reason) {
     237       44317 :         Cand = TopCand;
     238             :       } else {
     239       90611 :         Cand = BotCand;
     240             :       }
     241             :     }
     242             :   }
     243             :   LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
     244             : 
     245      270367 :   IsTopNode = Cand.AtTop;
     246      270367 :   return Cand.SU;
     247             : }
     248             : 
     249             : // This function is mostly cut and pasted from
     250             : // GenericScheduler::pickNode()
     251      362888 : SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
     252      362888 :   if (DAG->top() == DAG->bottom()) {
     253             :     assert(Top.Available.empty() && Top.Pending.empty() &&
     254             :            Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
     255             :     return nullptr;
     256             :   }
     257             :   SUnit *SU;
     258             :   do {
     259      341436 :     if (RegionPolicy.OnlyTopDown) {
     260           0 :       SU = Top.pickOnlyChoice();
     261           0 :       if (!SU) {
     262           0 :         CandPolicy NoPolicy;
     263           0 :         TopCand.reset(NoPolicy);
     264           0 :         pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
     265             :         assert(TopCand.Reason != NoCand && "failed to find a candidate");
     266           0 :         SU = TopCand.SU;
     267             :       }
     268           0 :       IsTopNode = true;
     269      341436 :     } else if (RegionPolicy.OnlyBottomUp) {
     270           0 :       SU = Bot.pickOnlyChoice();
     271           0 :       if (!SU) {
     272           0 :         CandPolicy NoPolicy;
     273           0 :         BotCand.reset(NoPolicy);
     274           0 :         pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
     275             :         assert(BotCand.Reason != NoCand && "failed to find a candidate");
     276           0 :         SU = BotCand.SU;
     277             :       }
     278           0 :       IsTopNode = false;
     279             :     } else {
     280      341436 :       SU = pickNodeBidirectional(IsTopNode);
     281             :     }
     282      341436 :   } while (SU->isScheduled);
     283             : 
     284      341436 :   if (SU->isTopReady())
     285      146623 :     Top.removeReady(SU);
     286      341436 :   if (SU->isBottomReady())
     287      294763 :     Bot.removeReady(SU);
     288             : 
     289             :   LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
     290             :                     << *SU->getInstr());
     291             :   return SU;
     292             : }
     293             : 
     294       19515 : GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
     295       19515 :                         std::unique_ptr<MachineSchedStrategy> S) :
     296             :   ScheduleDAGMILive(C, std::move(S)),
     297       19515 :   ST(MF.getSubtarget<GCNSubtarget>()),
     298       19515 :   MFI(*MF.getInfo<SIMachineFunctionInfo>()),
     299       19515 :   StartingOccupancy(MFI.getOccupancy()),
     300       58545 :   MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
     301             : 
     302             :   LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
     303       19515 : }
     304             : 
     305       42556 : void GCNScheduleDAGMILive::schedule() {
     306       42556 :   if (Stage == 0) {
     307             :     // Just record regions at the first pass.
     308       42216 :     Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
     309       42544 :     return;
     310             :   }
     311             : 
     312             :   std::vector<MachineInstr*> Unsched;
     313       21448 :   Unsched.reserve(NumRegionInstrs);
     314      361694 :   for (auto &I : *this) {
     315      340246 :     Unsched.push_back(&I);
     316             :   }
     317             : 
     318             :   GCNRegPressure PressureBefore;
     319       21448 :   if (LIS) {
     320       42896 :     PressureBefore = Pressure[RegionIdx];
     321             : 
     322             :     LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
     323             :                GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
     324             :                dbgs() << "Region live-in pressure:  ";
     325             :                llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
     326             :                dbgs() << "Region register pressure: ";
     327             :                PressureBefore.print(dbgs()));
     328             :   }
     329             : 
     330       21448 :   ScheduleDAGMILive::schedule();
     331       21448 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     332             : 
     333       21448 :   if (!LIS)
     334             :     return;
     335             : 
     336             :   // Check the results of scheduling.
     337             :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     338       21448 :   auto PressureAfter = getRealRegPressure();
     339             : 
     340             :   LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";
     341             :              PressureAfter.print(dbgs()));
     342             : 
     343       21448 :   if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
     344       21331 :       PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
     345       20836 :     Pressure[RegionIdx] = PressureAfter;
     346             :     LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
     347       20836 :     return;
     348             :   }
     349         612 :   unsigned Occ = MFI.getOccupancy();
     350         612 :   unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
     351         875 :   unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
     352             :   LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
     353             :                     << ", after " << WavesAfter << ".\n");
     354             : 
     355             :   // We could not keep current target occupancy because of the just scheduled
     356             :   // region. Record new occupancy for next scheduling cycle.
     357         612 :   unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
     358             :   // Allow memory bound functions to drop to 4 waves if not limited by an
     359             :   // attribute.
     360         648 :   if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
     361          36 :       WavesAfter >= MFI.getMinAllowedOccupancy()) {
     362             :     LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "
     363             :                       << MFI.getMinAllowedOccupancy() << " waves\n");
     364             :     NewOccupancy = WavesAfter;
     365             :   }
     366         612 :   if (NewOccupancy < MinOccupancy) {
     367         252 :     MinOccupancy = NewOccupancy;
     368         252 :     MFI.limitOccupancy(MinOccupancy);
     369             :     LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
     370             :                       << MinOccupancy << ".\n");
     371             :   }
     372             : 
     373         612 :   if (WavesAfter >= MinOccupancy) {
     374         600 :     Pressure[RegionIdx] = PressureAfter;
     375         600 :     return;
     376             :   }
     377             : 
     378             :   LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n");
     379          12 :   RegionEnd = RegionBegin;
     380        1216 :   for (MachineInstr *MI : Unsched) {
     381             :     if (MI->isDebugInstr())
     382           0 :       continue;
     383             : 
     384        2408 :     if (MI->getIterator() != RegionEnd) {
     385         665 :       BB->remove(MI);
     386         665 :       BB->insert(RegionEnd, MI);
     387             :       if (!MI->isDebugInstr())
     388         665 :         LIS->handleMove(*MI, true);
     389             :     }
     390             :     // Reset read-undef flags and update them later.
     391        7318 :     for (auto &Op : MI->operands())
     392        6114 :       if (Op.isReg() && Op.isDef())
     393             :         Op.setIsUndef(false);
     394        1204 :     RegisterOperands RegOpers;
     395        1204 :     RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
     396             :     if (!MI->isDebugInstr()) {
     397        1204 :       if (ShouldTrackLaneMasks) {
     398             :         // Adjust liveness and add missing dead+read-undef flags.
     399        1204 :         SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
     400        1204 :         RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
     401             :       } else {
     402             :         // Adjust for missing dead-def flags.
     403           0 :         RegOpers.detectDeadDefs(*MI, *LIS);
     404             :       }
     405             :     }
     406        1204 :     RegionEnd = MI->getIterator();
     407             :     ++RegionEnd;
     408             :     LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
     409             :   }
     410          12 :   RegionBegin = Unsched.front()->getIterator();
     411          12 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     412             : 
     413          12 :   placeDebugValues();
     414             : }
     415             : 
     416       21448 : GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
     417       21448 :   GCNDownwardRPTracker RPTracker(*LIS);
     418       42896 :   RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
     419       21448 :   return RPTracker.moveMaxPressure();
     420             : }
     421             : 
     422       20403 : void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
     423       20403 :   GCNDownwardRPTracker RPTracker(*LIS);
     424             : 
     425             :   // If the block has the only successor then live-ins of that successor are
     426             :   // live-outs of the current block. We can reuse calculated live set if the
     427             :   // successor will be sent to scheduling past current block.
     428       20403 :   const MachineBasicBlock *OnlySucc = nullptr;
     429       20403 :   if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
     430         950 :     SlotIndexes *Ind = LIS->getSlotIndexes();
     431         950 :     if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
     432         875 :       OnlySucc = *MBB->succ_begin();
     433             :   }
     434             : 
     435             :   // Scheduler sends regions from the end of the block upwards.
     436       20403 :   size_t CurRegion = RegionIdx;
     437       41511 :   for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
     438       22688 :     if (Regions[CurRegion].first->getParent() != MBB)
     439             :       break;
     440       20403 :   --CurRegion;
     441             : 
     442             :   auto I = MBB->begin();
     443       20403 :   auto LiveInIt = MBBLiveIns.find(MBB);
     444       20403 :   if (LiveInIt != MBBLiveIns.end()) {
     445             :     auto LiveIn = std::move(LiveInIt->second);
     446         570 :     RPTracker.reset(*MBB->begin(), &LiveIn);
     447             :     MBBLiveIns.erase(LiveInIt);
     448             :   } else {
     449             :     I = Regions[CurRegion].first;
     450       19833 :     RPTracker.reset(*I);
     451             :   }
     452             : 
     453             :   for ( ; ; ) {
     454      312607 :     I = RPTracker.getNext();
     455             : 
     456      333010 :     if (Regions[CurRegion].first == I) {
     457             :       LiveIns[CurRegion] = RPTracker.getLiveRegs();
     458             :       RPTracker.clearMaxPressure();
     459             :     }
     460             : 
     461      333010 :     if (Regions[CurRegion].second == I) {
     462       21108 :       Pressure[CurRegion] = RPTracker.moveMaxPressure();
     463       21108 :       if (CurRegion-- == RegionIdx)
     464             :         break;
     465             :     }
     466      312607 :     RPTracker.advanceToNext();
     467      312607 :     RPTracker.advanceBeforeNext();
     468             :   }
     469             : 
     470       20403 :   if (OnlySucc) {
     471         875 :     if (I != MBB->end()) {
     472         173 :       RPTracker.advanceToNext();
     473         173 :       RPTracker.advance(MBB->end());
     474             :     }
     475        1750 :     RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
     476         875 :     RPTracker.advanceBeforeNext();
     477        1750 :     MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
     478             :   }
     479       20403 : }
     480             : 
     481       19515 : void GCNScheduleDAGMILive::finalizeSchedule() {
     482             :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     483             :   LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
     484             : 
     485       39030 :   LiveIns.resize(Regions.size());
     486       39030 :   Pressure.resize(Regions.size());
     487             : 
     488             :   do {
     489       39030 :     Stage++;
     490       39030 :     RegionIdx = 0;
     491             :     MachineBasicBlock *MBB = nullptr;
     492             : 
     493       39030 :     if (Stage > 1) {
     494             :       // Retry function scheduling if we found resulting occupancy and it is
     495             :       // lower than used for first pass scheduling. This will give more freedom
     496             :       // to schedule low register pressure blocks.
     497             :       // Code is partially copied from MachineSchedulerBase::scheduleRegions().
     498             : 
     499       19515 :       if (!LIS || StartingOccupancy <= MinOccupancy)
     500             :         break;
     501             : 
     502             :       LLVM_DEBUG(
     503             :           dbgs()
     504             :           << "Retrying function scheduling with lowest recorded occupancy "
     505             :           << MinOccupancy << ".\n");
     506             : 
     507             :       S.setTargetOccupancy(MinOccupancy);
     508             :     }
     509             : 
     510       41213 :     for (auto Region : Regions) {
     511       21448 :       RegionBegin = Region.first;
     512       21448 :       RegionEnd = Region.second;
     513             : 
     514       21448 :       if (RegionBegin->getParent() != MBB) {
     515       20722 :         if (MBB) finishBlock();
     516       20722 :         MBB = RegionBegin->getParent();
     517       20722 :         startBlock(MBB);
     518       20722 :         if (Stage == 1)
     519       20403 :           computeBlockPressure(MBB);
     520             :       }
     521             : 
     522       21448 :       unsigned NumRegionInstrs = std::distance(begin(), end());
     523       21448 :       enterRegion(MBB, begin(), end(), NumRegionInstrs);
     524             : 
     525             :       // Skip empty scheduling regions (0 or 1 schedulable instructions).
     526       42896 :       if (begin() == end() || begin() == std::prev(end())) {
     527           0 :         exitRegion();
     528           0 :         continue;
     529             :       }
     530             : 
     531             :       LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
     532             :       LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "
     533             :                         << MBB->getName() << "\n  From: " << *begin()
     534             :                         << "    To: ";
     535             :                  if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
     536             :                  else dbgs() << "End";
     537             :                  dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
     538             : 
     539       21448 :       schedule();
     540             : 
     541       21448 :       exitRegion();
     542       21448 :       ++RegionIdx;
     543             :     }
     544       19765 :     finishBlock();
     545             : 
     546       19765 :   } while (Stage < 2);
     547       19515 : }

Generated by: LCOV version 1.13