LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - GCNSchedStrategy.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 247 264 93.6 %
Date: 2017-09-14 15:23:50 Functions: 12 12 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This contains a MachineSchedStrategy implementation for maximizing wave
      12             : /// occupancy on GCN hardware.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "GCNSchedStrategy.h"
      16             : #include "AMDGPUSubtarget.h"
      17             : #include "SIInstrInfo.h"
      18             : #include "SIMachineFunctionInfo.h"
      19             : #include "SIRegisterInfo.h"
      20             : #include "llvm/CodeGen/RegisterClassInfo.h"
      21             : #include "llvm/Support/MathExtras.h"
      22             : 
      23             : #define DEBUG_TYPE "machine-scheduler"
      24             : 
      25             : using namespace llvm;
      26             : 
      27       14650 : GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
      28       14650 :     const MachineSchedContext *C) :
      29       14650 :     GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
      30             : 
      31         960 : static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs,
      32             :                             const MachineFunction &MF) {
      33             : 
      34         960 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
      35         960 :   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
      36        2880 :   unsigned MinRegOccupancy = std::min(ST.getOccupancyWithNumSGPRs(SGPRs),
      37        3840 :                                       ST.getOccupancyWithNumVGPRs(VGPRs));
      38             :   return std::min(MinRegOccupancy,
      39        2880 :                   ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
      40        2880 :                                                   *MF.getFunction()));
      41             : }
      42             : 
      43       15984 : void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
      44       15984 :   GenericScheduler::initialize(DAG);
      45             : 
      46       15984 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
      47             : 
      48       15984 :   MF = &DAG->MF;
      49             : 
      50       15984 :   const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
      51             : 
      52             :   // FIXME: This is also necessary, because some passes that run after
      53             :   // scheduling and before regalloc increase register pressure.
      54       15984 :   const int ErrorMargin = 3;
      55             : 
      56       31968 :   SGPRExcessLimit = Context->RegClassInfo
      57       15984 :     ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
      58       31968 :   VGPRExcessLimit = Context->RegClassInfo
      59       15984 :     ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
      60       15984 :   if (TargetOccupancy) {
      61         526 :     SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
      62         526 :     VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
      63             :   } else {
      64       15721 :     SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      65             :                                                     SRI->getSGPRPressureSet());
      66       15721 :     VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      67             :                                                     SRI->getVGPRPressureSet());
      68             :   }
      69             : 
      70       15984 :   SGPRCriticalLimit -= ErrorMargin;
      71       15984 :   VGPRCriticalLimit -= ErrorMargin;
      72       15984 : }
      73             : 
      74     2827253 : void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
      75             :                                      bool AtTop, const RegPressureTracker &RPTracker,
      76             :                                      const SIRegisterInfo *SRI,
      77             :                                      unsigned SGPRPressure,
      78             :                                      unsigned VGPRPressure) {
      79             : 
      80     2827253 :   Cand.SU = SU;
      81     2827253 :   Cand.AtTop = AtTop;
      82             : 
      83             :   // getDownwardPressure() and getUpwardPressure() make temporary changes to
      84             :   // the the tracker, so we need to pass those function a non-const copy.
      85     2827253 :   RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
      86             : 
      87     5654506 :   std::vector<unsigned> Pressure;
      88     5654506 :   std::vector<unsigned> MaxPressure;
      89             : 
      90     2827253 :   if (AtTop)
      91      416030 :     TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
      92             :   else {
      93             :     // FIXME: I think for bottom up scheduling, the register pressure is cached
      94             :     // and can be retrieved by DAG->getPressureDif(SU).
      95     2411223 :     TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
      96             :   }
      97             : 
      98     5654506 :   unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
      99     5654506 :   unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
     100             : 
     101             :   // If two instructions increase the pressure of different register sets
     102             :   // by the same amount, the generic scheduler will prefer to schedule the
     103             :   // instruction that increases the set with the least amount of registers,
     104             :   // which in our case would be SGPRs.  This is rarely what we want, so
     105             :   // when we report excess/critical register pressure, we do it either
     106             :   // only for VGPRs or only for SGPRs.
     107             : 
     108             :   // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
     109     2827253 :   const unsigned MaxVGPRPressureInc = 16;
     110     2827253 :   bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
     111     2827253 :   bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
     112             : 
     113             : 
     114             :   // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
     115             :   // to increase the likelihood we don't go over the limits.  We should improve
     116             :   // the analysis to look through dependencies to find the path with the least
     117             :   // register pressure.
     118             : 
     119             :   // We only need to update the RPDelata for instructions that increase
     120             :   // register pressure.  Instructions that decrease or keep reg pressure
     121             :   // the same will be marked as RegExcess in tryCandidate() when they
     122             :   // are compared with instructions that increase the register pressure.
     123     2827253 :   if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
     124       18394 :     Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
     125        9197 :     Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
     126             :   }
     127             : 
     128     2827253 :   if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
     129      135112 :     Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
     130       67556 :     Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
     131             :   }
     132             : 
     133             :   // Register pressure is considered 'CRITICAL' if it is approaching a value
     134             :   // that would reduce the wave occupancy for the execution unit.  When
     135             :   // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
     136             :   // has the same cost, so we don't need to prefer one over the other.
     137             : 
     138     2827253 :   int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
     139     2827253 :   int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
     140             : 
     141     2827253 :   if (SGPRDelta >= 0 || VGPRDelta >= 0) {
     142      935405 :     if (SGPRDelta > VGPRDelta) {
     143      294434 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
     144      147217 :       Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
     145             :     } else {
     146     1576376 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
     147      788188 :       Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
     148             :     }
     149             :   }
     150     2827253 : }
     151             : 
     152             : // This function is mostly cut and pasted from
     153             : // GenericScheduler::pickNodeFromQueue()
     154      258663 : void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
     155             :                                          const CandPolicy &ZonePolicy,
     156             :                                          const RegPressureTracker &RPTracker,
     157             :                                          SchedCandidate &Cand) {
     158      258663 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
     159      517326 :   ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
     160      517326 :   unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
     161      517326 :   unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
     162      258663 :   ReadyQueue &Q = Zone.Available;
     163     3861905 :   for (SUnit *SU : Q) {
     164             : 
     165     2827253 :     SchedCandidate TryCand(ZonePolicy);
     166     2827253 :     initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
     167             :                   SGPRPressure, VGPRPressure);
     168             :     // Pass SchedBoundary only when comparing nodes from the same boundary.
     169     2827253 :     SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
     170     2827253 :     GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
     171     2827253 :     if (TryCand.Reason != NoCand) {
     172             :       // Initialize resource delta if needed in case future heuristics query it.
     173     1959800 :       if (TryCand.ResDelta == SchedResourceDelta())
     174      979900 :         TryCand.initResourceDelta(Zone.DAG, SchedModel);
     175             :       Cand.setBest(TryCand);
     176             :     }
     177             :   }
     178      258663 : }
     179             : 
     180             : // This function is mostly cut and pasted from
     181             : // GenericScheduler::pickNodeBidirectional()
     182      266879 : SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
     183             :   // Schedule as far as possible in the direction of no choice. This is most
     184             :   // efficient, but also provides the best heuristics for CriticalPSets.
     185      266879 :   if (SUnit *SU = Bot.pickOnlyChoice()) {
     186       52188 :     IsTopNode = false;
     187       52188 :     return SU;
     188             :   }
     189      214691 :   if (SUnit *SU = Top.pickOnlyChoice()) {
     190        5033 :     IsTopNode = true;
     191        5033 :     return SU;
     192             :   }
     193             :   // Set the bottom-up policy based on the state of the current bottom zone and
     194             :   // the instructions outside the zone, including the top zone.
     195      209658 :   CandPolicy BotPolicy;
     196      209658 :   setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
     197             :   // Set the top-down policy based on the state of the current top zone and
     198             :   // the instructions outside the zone, including the bottom zone.
     199      209658 :   CandPolicy TopPolicy;
     200      209658 :   setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
     201             : 
     202             :   // See if BotCand is still valid (because we previously scheduled from Top).
     203             :   DEBUG(dbgs() << "Picking from Bot:\n");
     204      209658 :   if (!BotCand.isValid() || BotCand.SU->isScheduled ||
     205       38551 :       BotCand.Policy != BotPolicy) {
     206      353118 :     BotCand.reset(CandPolicy());
     207      353118 :     pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
     208             :     assert(BotCand.Reason != NoCand && "failed to find the first candidate");
     209             :   } else {
     210             :     DEBUG(traceCandidate(BotCand));
     211             :   }
     212             : 
     213             :   // Check if the top Q has a better candidate.
     214             :   DEBUG(dbgs() << "Picking from Top:\n");
     215      209658 :   if (!TopCand.isValid() || TopCand.SU->isScheduled ||
     216      152107 :       TopCand.Policy != TopPolicy) {
     217      164208 :     TopCand.reset(CandPolicy());
     218      164208 :     pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
     219             :     assert(TopCand.Reason != NoCand && "failed to find the first candidate");
     220             :   } else {
     221             :     DEBUG(traceCandidate(TopCand));
     222             :   }
     223             : 
     224             :   // Pick best from BotCand and TopCand.
     225             :   DEBUG(
     226             :     dbgs() << "Top Cand: ";
     227             :     traceCandidate(TopCand);
     228             :     dbgs() << "Bot Cand: ";
     229             :     traceCandidate(BotCand);
     230             :   );
     231      209658 :   SchedCandidate Cand;
     232      209658 :   if (TopCand.Reason == BotCand.Reason) {
     233       86879 :     Cand = BotCand;
     234       86879 :     GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
     235       86879 :     TopCand.Reason = NoCand;
     236       86879 :     GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
     237       86879 :     if (TopCand.Reason != NoCand) {
     238        4766 :       Cand.setBest(TopCand);
     239             :     } else {
     240       82113 :       TopCand.Reason = TopReason;
     241             :     }
     242             :   } else {
     243      122779 :     if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
     244           1 :       Cand = TopCand;
     245      123390 :     } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
     246         230 :       Cand = BotCand;
     247      123203 :     } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     248         173 :       Cand = TopCand;
     249      138520 :     } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     250        9908 :       Cand = BotCand;
     251             :     } else {
     252      112467 :       if (BotCand.Reason > TopCand.Reason) {
     253       37415 :         Cand = TopCand;
     254             :       } else {
     255       75052 :         Cand = BotCand;
     256             :       }
     257             :     }
     258             :   }
     259             :   DEBUG(
     260             :     dbgs() << "Picking: ";
     261             :     traceCandidate(Cand);
     262             :   );
     263             : 
     264      209658 :   IsTopNode = Cand.AtTop;
     265      209658 :   return Cand.SU;
     266             : }
     267             : 
     268             : // This function is mostly cut and pasted from
     269             : // GenericScheduler::pickNode()
     270      282863 : SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
     271      565726 :   if (DAG->top() == DAG->bottom()) {
     272             :     assert(Top.Available.empty() && Top.Pending.empty() &&
     273             :            Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
     274             :     return nullptr;
     275             :   }
     276             :   SUnit *SU;
     277             :   do {
     278      266879 :     if (RegionPolicy.OnlyTopDown) {
     279           0 :       SU = Top.pickOnlyChoice();
     280           0 :       if (!SU) {
     281           0 :         CandPolicy NoPolicy;
     282           0 :         TopCand.reset(NoPolicy);
     283           0 :         pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
     284             :         assert(TopCand.Reason != NoCand && "failed to find a candidate");
     285           0 :         SU = TopCand.SU;
     286             :       }
     287           0 :       IsTopNode = true;
     288      266879 :     } else if (RegionPolicy.OnlyBottomUp) {
     289           0 :       SU = Bot.pickOnlyChoice();
     290           0 :       if (!SU) {
     291           0 :         CandPolicy NoPolicy;
     292           0 :         BotCand.reset(NoPolicy);
     293           0 :         pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
     294             :         assert(BotCand.Reason != NoCand && "failed to find a candidate");
     295           0 :         SU = BotCand.SU;
     296             :       }
     297           0 :       IsTopNode = false;
     298             :     } else {
     299      266879 :       SU = pickNodeBidirectional(IsTopNode);
     300             :     }
     301      266879 :   } while (SU->isScheduled);
     302             : 
     303      266879 :   if (SU->isTopReady())
     304      113678 :     Top.removeReady(SU);
     305      266879 :   if (SU->isBottomReady())
     306      229471 :     Bot.removeReady(SU);
     307             : 
     308             :   DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr());
     309             :   return SU;
     310             : }
     311             : 
     312       14647 : GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
     313       14647 :                         std::unique_ptr<MachineSchedStrategy> S) :
     314       14647 :   ScheduleDAGMILive(C, std::move(S)),
     315       14647 :   ST(MF.getSubtarget<SISubtarget>()),
     316       14647 :   MFI(*MF.getInfo<SIMachineFunctionInfo>()),
     317       29294 :   StartingOccupancy(ST.getOccupancyWithLocalMemSize(MFI.getLDSSize(),
     318       14647 :                                                     *MF.getFunction())),
     319      146470 :   MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
     320             : 
     321             :   DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
     322       14647 : }
     323             : 
     324       31697 : void GCNScheduleDAGMILive::schedule() {
     325       31697 :   if (Stage == 0) {
     326             :     // Just record regions at the first pass.
     327       31436 :     Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
     328       47380 :     return;
     329             :   }
     330             : 
     331       16014 :   std::vector<MachineInstr*> Unsched;
     332       15979 :   Unsched.reserve(NumRegionInstrs);
     333      562298 :   for (auto &I : *this)
     334      530340 :     Unsched.push_back(&I);
     335             : 
     336       15979 :   GCNRegPressure PressureBefore;
     337       15979 :   if (LIS) {
     338       31958 :     PressureBefore = Pressure[RegionIdx];
     339             : 
     340             :     DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
     341             :           GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
     342             :           dbgs() << "Region live-in pressure:  ";
     343             :           llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
     344             :           dbgs() << "Region register pressure: ";
     345             :           PressureBefore.print(dbgs()));
     346             :   }
     347             : 
     348       15979 :   ScheduleDAGMILive::schedule();
     349       63916 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     350             : 
     351       15979 :   if (!LIS)
     352             :     return;
     353             : 
     354             :   // Check the results of scheduling.
     355       31958 :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     356       15979 :   auto PressureAfter = getRealRegPressure();
     357             : 
     358             :   DEBUG(dbgs() << "Pressure after scheduling: "; PressureAfter.print(dbgs()));
     359             : 
     360       31850 :   if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
     361       15871 :       PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
     362       30998 :     Pressure[RegionIdx] = PressureAfter;
     363             :     DEBUG(dbgs() << "Pressure in desired limits, done.\n");
     364       15499 :     return;
     365             :   }
     366         480 :   unsigned WavesAfter = getMaxWaves(PressureAfter.getSGPRNum(),
     367         960 :                                     PressureAfter.getVGPRNum(), MF);
     368         480 :   unsigned WavesBefore = getMaxWaves(PressureBefore.getSGPRNum(),
     369         960 :                                      PressureBefore.getVGPRNum(), MF);
     370             :   DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore <<
     371             :                   ", after " << WavesAfter << ".\n");
     372             : 
     373             :   // We could not keep current target occupancy because of the just scheduled
     374             :   // region. Record new occupancy for next scheduling cycle.
     375         480 :   unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
     376         480 :   if (NewOccupancy < MinOccupancy) {
     377         184 :     MinOccupancy = NewOccupancy;
     378             :     DEBUG(dbgs() << "Occupancy lowered for the function to "
     379             :                  << MinOccupancy << ".\n");
     380             :   }
     381             : 
     382         480 :   if (WavesAfter >= WavesBefore) {
     383         890 :     Pressure[RegionIdx] = PressureAfter;
     384         445 :     return;
     385             :   }
     386             : 
     387             :   DEBUG(dbgs() << "Attempting to revert scheduling.\n");
     388          35 :   RegionEnd = RegionBegin;
     389        4618 :   for (MachineInstr *MI : Unsched) {
     390       13434 :     if (MI->getIterator() != RegionEnd) {
     391        5250 :       BB->remove(MI);
     392        5250 :       BB->insert(RegionEnd, MI);
     393        2625 :       LIS->handleMove(*MI, true);
     394             :     }
     395             :     // Reset read-undef flags and update them later.
     396       23863 :     for (auto &Op : MI->operands())
     397       33902 :       if (Op.isReg() && Op.isDef())
     398             :         Op.setIsUndef(false);
     399        8956 :     RegisterOperands RegOpers;
     400        4478 :     RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
     401        4478 :     if (ShouldTrackLaneMasks) {
     402             :       // Adjust liveness and add missing dead+read-undef flags.
     403       13434 :       SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
     404        4478 :       RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
     405             :     } else {
     406             :       // Adjust for missing dead-def flags.
     407           0 :       RegOpers.detectDeadDefs(*MI, *LIS);
     408             :     }
     409        8956 :     RegionEnd = MI->getIterator();
     410        8956 :     ++RegionEnd;
     411             :     DEBUG(dbgs() << "Scheduling " << *MI);
     412             :   }
     413          70 :   RegionBegin = Unsched.front()->getIterator();
     414         140 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     415             : 
     416          35 :   placeDebugValues();
     417             : }
     418             : 
     419       15979 : GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
     420       47937 :   GCNDownwardRPTracker RPTracker(*LIS);
     421       63916 :   RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
     422       31958 :   return RPTracker.moveMaxPressure();
     423             : }
     424             : 
     425       15282 : void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
     426       45846 :   GCNDownwardRPTracker RPTracker(*LIS);
     427             : 
     428             :   // If the block has the only successor then live-ins of that successor are
     429             :   // live-outs of the current block. We can reuse calculated live set if the
     430             :   // successor will be sent to scheduling past current block.
     431       15282 :   const MachineBasicBlock *OnlySucc = nullptr;
     432       16784 :   if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
     433         748 :     SlotIndexes *Ind = LIS->getSlotIndexes();
     434        2992 :     if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
     435         678 :       OnlySucc = *MBB->succ_begin();
     436             :   }
     437             : 
     438             :   // Scheduler sends regions from the end of the block upwards.
     439       15282 :   size_t CurRegion = RegionIdx;
     440       46282 :   for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
     441       50838 :     if (Regions[CurRegion].first->getParent() != MBB)
     442             :       break;
     443       15282 :   --CurRegion;
     444             : 
     445       15282 :   auto I = MBB->begin();
     446       15282 :   auto LiveInIt = MBBLiveIns.find(MBB);
     447       45846 :   if (LiveInIt != MBBLiveIns.end()) {
     448        1224 :     auto LiveIn = std::move(LiveInIt->second);
     449         816 :     RPTracker.reset(*MBB->begin(), &LiveIn);
     450         816 :     MBBLiveIns.erase(LiveInIt);
     451             :   } else {
     452       44622 :     I = Regions[CurRegion].first;
     453       14874 :     RPTracker.reset(*I);
     454             :   }
     455             : 
     456             :   for ( ; ; ) {
     457      245069 :     I = RPTracker.getNext();
     458             : 
     459     1041404 :     if (Regions[CurRegion].first == I) {
     460       31434 :       LiveIns[CurRegion] = RPTracker.getLiveRegs();
     461             :       RPTracker.clearMaxPressure();
     462             :     }
     463             : 
     464     1041404 :     if (Regions[CurRegion].second == I) {
     465       47154 :       Pressure[CurRegion] = RPTracker.moveMaxPressure();
     466       15718 :       if (CurRegion-- == RegionIdx)
     467             :         break;
     468             :     }
     469      245069 :     RPTracker.advanceToNext();
     470      245069 :     RPTracker.advanceBeforeNext();
     471             :   }
     472             : 
     473       15282 :   if (OnlySucc) {
     474        1356 :     if (I != MBB->end()) {
     475         168 :       RPTracker.advanceToNext();
     476         168 :       RPTracker.advance(MBB->end());
     477             :     }
     478        2034 :     RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
     479         678 :     RPTracker.advanceBeforeNext();
     480        2034 :     MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
     481             :   }
     482       15282 : }
     483             : 
     484       14647 : void GCNScheduleDAGMILive::finalizeSchedule() {
     485       29294 :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     486             :   DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
     487             : 
     488       29294 :   LiveIns.resize(Regions.size());
     489       29294 :   Pressure.resize(Regions.size());
     490             : 
     491             :   do {
     492       29294 :     Stage++;
     493       29294 :     RegionIdx = 0;
     494       29294 :     MachineBasicBlock *MBB = nullptr;
     495             : 
     496       29294 :     if (Stage > 1) {
     497             :       // Retry function scheduling if we found resulting occupancy and it is
     498             :       // lower than used for first pass scheduling. This will give more freedom
     499             :       // to schedule low register pressure blocks.
     500             :       // Code is partially copied from MachineSchedulerBase::scheduleRegions().
     501             : 
     502       14647 :       if (!LIS || StartingOccupancy <= MinOccupancy)
     503             :         break;
     504             : 
     505             :       DEBUG(dbgs()
     506             :               << "Retrying function scheduling with lowest recorded occupancy "
     507             :               << MinOccupancy << ".\n");
     508             : 
     509         181 :       S.setTargetOccupancy(MinOccupancy);
     510             :     }
     511             : 
     512       60463 :     for (auto Region : Regions) {
     513       15979 :       RegionBegin = Region.first;
     514       15979 :       RegionEnd = Region.second;
     515             : 
     516       31958 :       if (RegionBegin->getParent() != MBB) {
     517       15520 :         if (MBB) finishBlock();
     518       31040 :         MBB = RegionBegin->getParent();
     519       15520 :         startBlock(MBB);
     520       15520 :         if (Stage == 1)
     521       15282 :           computeBlockPressure(MBB);
     522             :       }
     523             : 
     524       31958 :       unsigned NumRegionInstrs = std::distance(begin(), end());
     525       15979 :       enterRegion(MBB, begin(), end(), NumRegionInstrs);
     526             : 
     527             :       // Skip empty scheduling regions (0 or 1 schedulable instructions).
     528       63916 :       if (begin() == end() || begin() == std::prev(end())) {
     529           0 :         exitRegion();
     530           0 :         continue;
     531             :       }
     532             : 
     533             :       DEBUG(dbgs() << "********** MI Scheduling **********\n");
     534             :       DEBUG(dbgs() << MF.getName()
     535             :             << ":BB#" << MBB->getNumber() << " " << MBB->getName()
     536             :             << "\n  From: " << *begin() << "    To: ";
     537             :             if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
     538             :             else dbgs() << "End";
     539             :             dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
     540             : 
     541       15979 :       schedule();
     542             : 
     543       15979 :       exitRegion();
     544       15979 :       ++RegionIdx;
     545             :     }
     546       14828 :     finishBlock();
     547             : 
     548       14828 :   } while (Stage < 2);
     549       14647 : }

Generated by: LCOV version 1.13