LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - GCNSchedStrategy.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 220 239 92.1 %
Date: 2018-05-20 00:06:23 Functions: 12 12 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This contains a MachineSchedStrategy implementation for maximizing wave
      12             : /// occupancy on GCN hardware.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "GCNSchedStrategy.h"
      16             : #include "AMDGPUSubtarget.h"
      17             : #include "SIInstrInfo.h"
      18             : #include "SIMachineFunctionInfo.h"
      19             : #include "SIRegisterInfo.h"
      20             : #include "llvm/CodeGen/RegisterClassInfo.h"
      21             : #include "llvm/Support/MathExtras.h"
      22             : 
      23             : #define DEBUG_TYPE "machine-scheduler"
      24             : 
      25             : using namespace llvm;
      26             : 
      27       17328 : GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
      28       17328 :     const MachineSchedContext *C) :
      29       17328 :     GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
      30             : 
      31        1146 : static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs,
      32             :                             const MachineFunction &MF) {
      33             : 
      34        1146 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
      35             :   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
      36        3438 :   unsigned MinRegOccupancy = std::min(ST.getOccupancyWithNumSGPRs(SGPRs),
      37        3438 :                                       ST.getOccupancyWithNumVGPRs(VGPRs));
      38             :   return std::min(MinRegOccupancy,
      39        2292 :                   ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
      40        2292 :                                                   MF.getFunction()));
      41             : }
      42             : 
      43       18773 : void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
      44       18773 :   GenericScheduler::initialize(DAG);
      45             : 
      46       18773 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
      47             : 
      48       18773 :   MF = &DAG->MF;
      49             : 
      50       18773 :   const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
      51             : 
      52             :   // FIXME: This is also necessary, because some passes that run after
      53             :   // scheduling and before regalloc increase register pressure.
      54             :   const int ErrorMargin = 3;
      55             : 
      56       37546 :   SGPRExcessLimit = Context->RegClassInfo
      57       18773 :     ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
      58       37546 :   VGPRExcessLimit = Context->RegClassInfo
      59       18773 :     ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
      60       18773 :   if (TargetOccupancy) {
      61         310 :     SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
      62         620 :     VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
      63             :   } else {
      64       18463 :     SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      65             :                                                     SRI->getSGPRPressureSet());
      66       18463 :     VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
      67             :                                                     SRI->getVGPRPressureSet());
      68             :   }
      69             : 
      70       18773 :   SGPRCriticalLimit -= ErrorMargin;
      71       18773 :   VGPRCriticalLimit -= ErrorMargin;
      72       18773 : }
      73             : 
      74     3031068 : void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
      75             :                                      bool AtTop, const RegPressureTracker &RPTracker,
      76             :                                      const SIRegisterInfo *SRI,
      77             :                                      unsigned SGPRPressure,
      78             :                                      unsigned VGPRPressure) {
      79             : 
      80     3031068 :   Cand.SU = SU;
      81     3031068 :   Cand.AtTop = AtTop;
      82             : 
      83             :   // getDownwardPressure() and getUpwardPressure() make temporary changes to
      84             :   // the tracker, so we need to pass those function a non-const copy.
      85             :   RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
      86             : 
      87             :   std::vector<unsigned> Pressure;
      88             :   std::vector<unsigned> MaxPressure;
      89             : 
      90     3031068 :   if (AtTop)
      91      500058 :     TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
      92             :   else {
      93             :     // FIXME: I think for bottom up scheduling, the register pressure is cached
      94             :     // and can be retrieved by DAG->getPressureDif(SU).
      95     2531010 :     TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
      96             :   }
      97             : 
      98     6062136 :   unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
      99     6062136 :   unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
     100             : 
     101             :   // If two instructions increase the pressure of different register sets
     102             :   // by the same amount, the generic scheduler will prefer to schedule the
     103             :   // instruction that increases the set with the least amount of registers,
     104             :   // which in our case would be SGPRs.  This is rarely what we want, so
     105             :   // when we report excess/critical register pressure, we do it either
     106             :   // only for VGPRs or only for SGPRs.
     107             : 
     108             :   // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
     109             :   const unsigned MaxVGPRPressureInc = 16;
     110     3031068 :   bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
     111     3031068 :   bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
     112             : 
     113             : 
     114             :   // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
     115             :   // to increase the likelihood we don't go over the limits.  We should improve
     116             :   // the analysis to look through dependencies to find the path with the least
     117             :   // register pressure.
     118             : 
     119             :   // We only need to update the RPDelata for instructions that increase
     120             :   // register pressure.  Instructions that decrease or keep reg pressure
     121             :   // the same will be marked as RegExcess in tryCandidate() when they
     122             :   // are compared with instructions that increase the register pressure.
     123     3031068 :   if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
     124       10127 :     Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
     125       10127 :     Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
     126             :   }
     127             : 
     128     3031068 :   if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
     129       60214 :     Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
     130       30107 :     Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
     131             :   }
     132             : 
     133             :   // Register pressure is considered 'CRITICAL' if it is approaching a value
     134             :   // that would reduce the wave occupancy for the execution unit.  When
     135             :   // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
     136             :   // has the same cost, so we don't need to prefer one over the other.
     137             : 
     138     3031068 :   int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
     139     3031068 :   int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
     140             : 
     141     3031068 :   if (SGPRDelta >= 0 || VGPRDelta >= 0) {
     142     1015771 :     if (SGPRDelta > VGPRDelta) {
     143      248926 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
     144             :       Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
     145             :     } else {
     146     1782616 :       Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
     147             :       Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
     148             :     }
     149             :   }
     150     3031068 : }
     151             : 
     152             : // This function is mostly cut and pasted from
     153             : // GenericScheduler::pickNodeFromQueue()
     154      288176 : void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
     155             :                                          const CandPolicy &ZonePolicy,
     156             :                                          const RegPressureTracker &RPTracker,
     157             :                                          SchedCandidate &Cand) {
     158      288176 :   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
     159             :   ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
     160      576352 :   unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
     161      576352 :   unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
     162             :   ReadyQueue &Q = Zone.Available;
     163     3319244 :   for (SUnit *SU : Q) {
     164             : 
     165             :     SchedCandidate TryCand(ZonePolicy);
     166     3031068 :     initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
     167             :                   SGPRPressure, VGPRPressure);
     168             :     // Pass SchedBoundary only when comparing nodes from the same boundary.
     169     3031068 :     SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
     170     3031068 :     GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
     171     3031068 :     if (TryCand.Reason != NoCand) {
     172             :       // Initialize resource delta if needed in case future heuristics query it.
     173             :       if (TryCand.ResDelta == SchedResourceDelta())
     174     1060140 :         TryCand.initResourceDelta(Zone.DAG, SchedModel);
     175             :       Cand.setBest(TryCand);
     176             :     }
     177             :   }
     178      288176 : }
     179             : 
     180             : // This function is mostly cut and pasted from
     181             : // GenericScheduler::pickNodeBidirectional()
     182      299814 : SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
     183             :   // Schedule as far as possible in the direction of no choice. This is most
     184             :   // efficient, but also provides the best heuristics for CriticalPSets.
     185      299814 :   if (SUnit *SU = Bot.pickOnlyChoice()) {
     186       57991 :     IsTopNode = false;
     187       57991 :     return SU;
     188             :   }
     189      241823 :   if (SUnit *SU = Top.pickOnlyChoice()) {
     190        6001 :     IsTopNode = true;
     191        6001 :     return SU;
     192             :   }
     193             :   // Set the bottom-up policy based on the state of the current bottom zone and
     194             :   // the instructions outside the zone, including the top zone.
     195      235822 :   CandPolicy BotPolicy;
     196      235822 :   setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
     197             :   // Set the top-down policy based on the state of the current top zone and
     198             :   // the instructions outside the zone, including the bottom zone.
     199      235822 :   CandPolicy TopPolicy;
     200      235822 :   setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
     201             : 
     202             :   // See if BotCand is still valid (because we previously scheduled from Top).
     203             :   LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
     204      235822 :   if (!BotCand.isValid() || BotCand.SU->isScheduled ||
     205             :       BotCand.Policy != BotPolicy) {
     206      192950 :     BotCand.reset(CandPolicy());
     207      385900 :     pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
     208             :     assert(BotCand.Reason != NoCand && "failed to find the first candidate");
     209             :   } else {
     210             :     LLVM_DEBUG(traceCandidate(BotCand));
     211             :   }
     212             : 
     213             :   // Check if the top Q has a better candidate.
     214             :   LLVM_DEBUG(dbgs() << "Picking from Top:\n");
     215      235822 :   if (!TopCand.isValid() || TopCand.SU->isScheduled ||
     216             :       TopCand.Policy != TopPolicy) {
     217       95226 :     TopCand.reset(CandPolicy());
     218      190452 :     pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
     219             :     assert(TopCand.Reason != NoCand && "failed to find the first candidate");
     220             :   } else {
     221             :     LLVM_DEBUG(traceCandidate(TopCand));
     222             :   }
     223             : 
     224             :   // Pick best from BotCand and TopCand.
     225             :   LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
     226             :              dbgs() << "Bot Cand: "; traceCandidate(BotCand););
     227             :   SchedCandidate Cand;
     228      235822 :   if (TopCand.Reason == BotCand.Reason) {
     229       95800 :     Cand = BotCand;
     230             :     GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
     231       95800 :     TopCand.Reason = NoCand;
     232       95800 :     GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
     233       95800 :     if (TopCand.Reason != NoCand) {
     234             :       Cand.setBest(TopCand);
     235             :     } else {
     236       85797 :       TopCand.Reason = TopReason;
     237             :     }
     238             :   } else {
     239      140022 :     if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
     240          23 :       Cand = TopCand;
     241      140484 :     } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
     242         178 :       Cand = BotCand;
     243      140674 :     } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     244         241 :       Cand = TopCand;
     245      160106 :     } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
     246       11575 :       Cand = BotCand;
     247             :     } else {
     248      128005 :       if (BotCand.Reason > TopCand.Reason) {
     249       44686 :         Cand = TopCand;
     250             :       } else {
     251       83319 :         Cand = BotCand;
     252             :       }
     253             :     }
     254             :   }
     255             :   LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
     256             : 
     257      235822 :   IsTopNode = Cand.AtTop;
     258      235822 :   return Cand.SU;
     259             : }
     260             : 
     261             : // This function is mostly cut and pasted from
     262             : // GenericScheduler::pickNode()
     263      318587 : SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
     264      318587 :   if (DAG->top() == DAG->bottom()) {
     265             :     assert(Top.Available.empty() && Top.Pending.empty() &&
     266             :            Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
     267             :     return nullptr;
     268             :   }
     269             :   SUnit *SU;
     270             :   do {
     271      299814 :     if (RegionPolicy.OnlyTopDown) {
     272           0 :       SU = Top.pickOnlyChoice();
     273           0 :       if (!SU) {
     274           0 :         CandPolicy NoPolicy;
     275           0 :         TopCand.reset(NoPolicy);
     276           0 :         pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
     277             :         assert(TopCand.Reason != NoCand && "failed to find a candidate");
     278           0 :         SU = TopCand.SU;
     279             :       }
     280           0 :       IsTopNode = true;
     281      299814 :     } else if (RegionPolicy.OnlyBottomUp) {
     282           0 :       SU = Bot.pickOnlyChoice();
     283           0 :       if (!SU) {
     284           0 :         CandPolicy NoPolicy;
     285           0 :         BotCand.reset(NoPolicy);
     286           0 :         pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
     287             :         assert(BotCand.Reason != NoCand && "failed to find a candidate");
     288           0 :         SU = BotCand.SU;
     289             :       }
     290           0 :       IsTopNode = false;
     291             :     } else {
     292      299814 :       SU = pickNodeBidirectional(IsTopNode);
     293             :     }
     294      299814 :   } while (SU->isScheduled);
     295             : 
     296      299814 :   if (SU->isTopReady())
     297      133711 :     Top.removeReady(SU);
     298      299814 :   if (SU->isBottomReady())
     299      254865 :     Bot.removeReady(SU);
     300             : 
     301             :   LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
     302             :                     << *SU->getInstr());
     303             :   return SU;
     304             : }
     305             : 
     306       17325 : GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
     307       17325 :                         std::unique_ptr<MachineSchedStrategy> S) :
     308             :   ScheduleDAGMILive(C, std::move(S)),
     309       17325 :   ST(MF.getSubtarget<SISubtarget>()),
     310       17325 :   MFI(*MF.getInfo<SIMachineFunctionInfo>()),
     311       69300 :   StartingOccupancy(std::min(ST.getOccupancyWithLocalMemSize(MFI.getLDSSize(),
     312       17325 :                                                              MF.getFunction()),
     313       34650 :                              MFI.getMaxWavesPerEU())),
     314      121275 :   MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
     315             : 
     316             :   LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
     317       17325 : }
     318             : 
     319       37228 : void GCNScheduleDAGMILive::schedule() {
     320       37228 :   if (Stage == 0) {
     321             :     // Just record regions at the first pass.
     322       36920 :     Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
     323       55639 :     return;
     324             :   }
     325             : 
     326             :   std::vector<MachineInstr*> Unsched;
     327       18768 :   Unsched.reserve(NumRegionInstrs);
     328      335738 :   for (auto &I : *this) {
     329      596404 :     Unsched.push_back(&I);
     330             :   }
     331             : 
     332             :   GCNRegPressure PressureBefore;
     333       18768 :   if (LIS) {
     334       37536 :     PressureBefore = Pressure[RegionIdx];
     335             : 
     336             :     LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
     337             :                GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
     338             :                dbgs() << "Region live-in pressure:  ";
     339             :                llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
     340             :                dbgs() << "Region register pressure: ";
     341             :                PressureBefore.print(dbgs()));
     342             :   }
     343             : 
     344       18768 :   ScheduleDAGMILive::schedule();
     345       18768 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     346             : 
     347       18768 :   if (!LIS)
     348             :     return;
     349             : 
     350             :   // Check the results of scheduling.
     351             :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     352       18768 :   auto PressureAfter = getRealRegPressure();
     353             : 
     354             :   LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";
     355             :              PressureAfter.print(dbgs()));
     356             : 
     357       37432 :   if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
     358       18664 :       PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
     359       36390 :     Pressure[RegionIdx] = PressureAfter;
     360             :     LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
     361       18195 :     return;
     362             :   }
     363         573 :   unsigned WavesAfter = getMaxWaves(PressureAfter.getSGPRNum(),
     364        1146 :                                     PressureAfter.getVGPRNum(), MF);
     365         573 :   unsigned WavesBefore = getMaxWaves(PressureBefore.getSGPRNum(),
     366        1146 :                                      PressureBefore.getVGPRNum(), MF);
     367        1146 :   WavesAfter = std::min(WavesAfter, MFI.getMaxWavesPerEU());
     368        1146 :   WavesBefore = std::min(WavesBefore, MFI.getMaxWavesPerEU());
     369             :   LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
     370             :                     << ", after " << WavesAfter << ".\n");
     371             : 
     372             :   // We could not keep current target occupancy because of the just scheduled
     373             :   // region. Record new occupancy for next scheduling cycle.
     374         573 :   unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
     375         573 :   if (NewOccupancy < MinOccupancy) {
     376         233 :     MinOccupancy = NewOccupancy;
     377             :     LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
     378             :                       << MinOccupancy << ".\n");
     379             :   }
     380             : 
     381         573 :   if (WavesAfter >= WavesBefore) {
     382        1048 :     Pressure[RegionIdx] = PressureAfter;
     383         524 :     return;
     384             :   }
     385             : 
     386             :   LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n");
     387          49 :   RegionEnd = RegionBegin;
     388        6286 :   for (MachineInstr *MI : Unsched) {
     389           0 :     if (MI->isDebugInstr())
     390           0 :       continue;
     391             : 
     392       12474 :     if (MI->getIterator() != RegionEnd) {
     393        3770 :       BB->remove(MI);
     394        3770 :       BB->insert(RegionEnd, MI);
     395             :       if (!MI->isDebugInstr())
     396        3770 :         LIS->handleMove(*MI, true);
     397             :     }
     398             :     // Reset read-undef flags and update them later.
     399       60631 :     for (auto &Op : MI->operands())
     400       47690 :       if (Op.isReg() && Op.isDef())
     401             :         Op.setIsUndef(false);
     402        6237 :     RegisterOperands RegOpers;
     403        6237 :     RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
     404             :     if (!MI->isDebugInstr()) {
     405        6237 :       if (ShouldTrackLaneMasks) {
     406             :         // Adjust liveness and add missing dead+read-undef flags.
     407       12474 :         SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
     408        6237 :         RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
     409             :       } else {
     410             :         // Adjust for missing dead-def flags.
     411           0 :         RegOpers.detectDeadDefs(*MI, *LIS);
     412             :       }
     413             :     }
     414        6237 :     RegionEnd = MI->getIterator();
     415             :     ++RegionEnd;
     416             :     LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
     417             :   }
     418          49 :   RegionBegin = Unsched.front()->getIterator();
     419          49 :   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
     420             : 
     421          49 :   placeDebugValues();
     422             : }
     423             : 
     424       18768 : GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
     425       18768 :   GCNDownwardRPTracker RPTracker(*LIS);
     426       37536 :   RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
     427       18768 :   return RPTracker.moveMaxPressure();
     428             : }
     429             : 
     430       17920 : void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
     431       17920 :   GCNDownwardRPTracker RPTracker(*LIS);
     432             : 
     433             :   // If the block has the only successor then live-ins of that successor are
     434             :   // live-outs of the current block. We can reuse calculated live set if the
     435             :   // successor will be sent to scheduling past current block.
     436       17920 :   const MachineBasicBlock *OnlySucc = nullptr;
     437       18737 :   if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
     438         814 :     SlotIndexes *Ind = LIS->getSlotIndexes();
     439         814 :     if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
     440         735 :       OnlySucc = *MBB->succ_begin();
     441             :   }
     442             : 
     443             :   // Scheduler sends regions from the end of the block upwards.
     444       17920 :   size_t CurRegion = RegionIdx;
     445       54840 :   for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
     446       19757 :     if (Regions[CurRegion].first->getParent() != MBB)
     447             :       break;
     448       17920 :   --CurRegion;
     449             : 
     450             :   auto I = MBB->begin();
     451       17920 :   auto LiveInIt = MBBLiveIns.find(MBB);
     452       17920 :   if (LiveInIt != MBBLiveIns.end()) {
     453             :     auto LiveIn = std::move(LiveInIt->second);
     454         424 :     RPTracker.reset(*MBB->begin(), &LiveIn);
     455             :     MBBLiveIns.erase(LiveInIt);
     456             :   } else {
     457             :     I = Regions[CurRegion].first;
     458       17496 :     RPTracker.reset(*I);
     459             :   }
     460             : 
     461             :   for ( ; ; ) {
     462      273698 :     I = RPTracker.getNext();
     463             : 
     464      291618 :     if (Regions[CurRegion].first == I) {
     465             :       LiveIns[CurRegion] = RPTracker.getLiveRegs();
     466             :       RPTracker.clearMaxPressure();
     467             :     }
     468             : 
     469      291618 :     if (Regions[CurRegion].second == I) {
     470       18460 :       Pressure[CurRegion] = RPTracker.moveMaxPressure();
     471       18460 :       if (CurRegion-- == RegionIdx)
     472             :         break;
     473             :     }
     474      273698 :     RPTracker.advanceToNext();
     475      273698 :     RPTracker.advanceBeforeNext();
     476             :   }
     477             : 
     478       17920 :   if (OnlySucc) {
     479         735 :     if (I != MBB->end()) {
     480         177 :       RPTracker.advanceToNext();
     481         177 :       RPTracker.advance(MBB->end());
     482             :     }
     483        1470 :     RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
     484         735 :     RPTracker.advanceBeforeNext();
     485        1470 :     MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
     486             :   }
     487       17920 : }
     488             : 
     489       17325 : void GCNScheduleDAGMILive::finalizeSchedule() {
     490             :   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
     491             :   LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
     492             : 
     493       34650 :   LiveIns.resize(Regions.size());
     494       34650 :   Pressure.resize(Regions.size());
     495             : 
     496             :   do {
     497       34650 :     Stage++;
     498       34650 :     RegionIdx = 0;
     499             :     MachineBasicBlock *MBB = nullptr;
     500             : 
     501       34650 :     if (Stage > 1) {
     502             :       // Retry function scheduling if we found resulting occupancy and it is
     503             :       // lower than used for first pass scheduling. This will give more freedom
     504             :       // to schedule low register pressure blocks.
     505             :       // Code is partially copied from MachineSchedulerBase::scheduleRegions().
     506             : 
     507       17325 :       if (!LIS || StartingOccupancy <= MinOccupancy)
     508             :         break;
     509             : 
     510             :       LLVM_DEBUG(
     511             :           dbgs()
     512             :           << "Retrying function scheduling with lowest recorded occupancy "
     513             :           << MinOccupancy << ".\n");
     514             : 
     515             :       S.setTargetOccupancy(MinOccupancy);
     516             :     }
     517             : 
     518       55091 :     for (auto Region : Regions) {
     519       18768 :       RegionBegin = Region.first;
     520       18768 :       RegionEnd = Region.second;
     521             : 
     522       18768 :       if (RegionBegin->getParent() != MBB) {
     523       18207 :         if (MBB) finishBlock();
     524       18207 :         MBB = RegionBegin->getParent();
     525       18207 :         startBlock(MBB);
     526       18207 :         if (Stage == 1)
     527       17920 :           computeBlockPressure(MBB);
     528             :       }
     529             : 
     530       18768 :       unsigned NumRegionInstrs = std::distance(begin(), end());
     531       18768 :       enterRegion(MBB, begin(), end(), NumRegionInstrs);
     532             : 
     533             :       // Skip empty scheduling regions (0 or 1 schedulable instructions).
     534       37536 :       if (begin() == end() || begin() == std::prev(end())) {
     535           0 :         exitRegion();
     536           0 :         continue;
     537             :       }
     538             : 
     539             :       LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
     540             :       LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "
     541             :                         << MBB->getName() << "\n  From: " << *begin()
     542             :                         << "    To: ";
     543             :                  if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
     544             :                  else dbgs() << "End";
     545             :                  dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
     546             : 
     547       18768 :       schedule();
     548             : 
     549       18768 :       exitRegion();
     550       18768 :       ++RegionIdx;
     551             :     }
     552       17555 :     finishBlock();
     553             : 
     554       17555 :   } while (Stage < 2);
     555       17325 : }

Generated by: LCOV version 1.13