Line data Source code
1 : //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : /// \file
11 : /// This contains a MachineSchedStrategy implementation for maximizing wave
12 : /// occupancy on GCN hardware.
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "GCNSchedStrategy.h"
16 : #include "AMDGPUSubtarget.h"
17 : #include "SIInstrInfo.h"
18 : #include "SIMachineFunctionInfo.h"
19 : #include "SIRegisterInfo.h"
20 : #include "llvm/CodeGen/RegisterClassInfo.h"
21 : #include "llvm/Support/MathExtras.h"
22 :
23 : #define DEBUG_TYPE "machine-scheduler"
24 :
25 : using namespace llvm;
26 :
27 19518 : GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
28 19518 : const MachineSchedContext *C) :
29 19518 : GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
30 :
31 21452 : void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
32 21452 : GenericScheduler::initialize(DAG);
33 :
34 21452 : const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
35 :
36 21452 : MF = &DAG->MF;
37 :
38 21452 : const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
39 :
40 : // FIXME: This is also necessary, because some passes that run after
41 : // scheduling and before regalloc increase register pressure.
42 : const int ErrorMargin = 3;
43 :
44 42904 : SGPRExcessLimit = Context->RegClassInfo
45 21452 : ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
46 42904 : VGPRExcessLimit = Context->RegClassInfo
47 21452 : ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
48 21452 : if (TargetOccupancy) {
49 341 : SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
50 341 : VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
51 : } else {
52 21111 : SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
53 : SRI->getSGPRPressureSet());
54 21111 : VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
55 : SRI->getVGPRPressureSet());
56 : }
57 :
58 21452 : SGPRCriticalLimit -= ErrorMargin;
59 21452 : VGPRCriticalLimit -= ErrorMargin;
60 21452 : }
61 :
62 3323073 : void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
63 : bool AtTop, const RegPressureTracker &RPTracker,
64 : const SIRegisterInfo *SRI,
65 : unsigned SGPRPressure,
66 : unsigned VGPRPressure) {
67 :
68 3323073 : Cand.SU = SU;
69 3323073 : Cand.AtTop = AtTop;
70 :
71 : // getDownwardPressure() and getUpwardPressure() make temporary changes to
72 : // the tracker, so we need to pass those function a non-const copy.
73 : RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
74 :
75 : std::vector<unsigned> Pressure;
76 : std::vector<unsigned> MaxPressure;
77 :
78 3323073 : if (AtTop)
79 569742 : TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
80 : else {
81 : // FIXME: I think for bottom up scheduling, the register pressure is cached
82 : // and can be retrieved by DAG->getPressureDif(SU).
83 2753331 : TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
84 : }
85 :
86 3323073 : unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
87 3323073 : unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
88 :
89 : // If two instructions increase the pressure of different register sets
90 : // by the same amount, the generic scheduler will prefer to schedule the
91 : // instruction that increases the set with the least amount of registers,
92 : // which in our case would be SGPRs. This is rarely what we want, so
93 : // when we report excess/critical register pressure, we do it either
94 : // only for VGPRs or only for SGPRs.
95 :
96 : // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
97 : const unsigned MaxVGPRPressureInc = 16;
98 3323073 : bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
99 3323073 : bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
100 :
101 :
102 : // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
103 : // to increase the likelihood we don't go over the limits. We should improve
104 : // the analysis to look through dependencies to find the path with the least
105 : // register pressure.
106 :
107 : // We only need to update the RPDelata for instructions that increase
108 : // register pressure. Instructions that decrease or keep reg pressure
109 : // the same will be marked as RegExcess in tryCandidate() when they
110 : // are compared with instructions that increase the register pressure.
111 3323073 : if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
112 10886 : Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
113 10886 : Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
114 : }
115 :
116 3323073 : if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
117 25416 : Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
118 25416 : Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
119 : }
120 :
121 : // Register pressure is considered 'CRITICAL' if it is approaching a value
122 : // that would reduce the wave occupancy for the execution unit. When
123 : // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
124 : // has the same cost, so we don't need to prefer one over the other.
125 :
126 3323073 : int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
127 3323073 : int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
128 :
129 3323073 : if (SGPRDelta >= 0 || VGPRDelta >= 0) {
130 1130348 : if (SGPRDelta > VGPRDelta) {
131 260248 : Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
132 : Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
133 : } else {
134 2000448 : Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
135 : Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
136 : }
137 : }
138 3323073 : }
139 :
140 : // This function is mostly cut and pasted from
141 : // GenericScheduler::pickNodeFromQueue()
142 314616 : void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
143 : const CandPolicy &ZonePolicy,
144 : const RegPressureTracker &RPTracker,
145 : SchedCandidate &Cand) {
146 314616 : const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
147 : ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
148 314616 : unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
149 629232 : unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
150 : ReadyQueue &Q = Zone.Available;
151 3637689 : for (SUnit *SU : Q) {
152 :
153 : SchedCandidate TryCand(ZonePolicy);
154 3323073 : initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
155 : SGPRPressure, VGPRPressure);
156 : // Pass SchedBoundary only when comparing nodes from the same boundary.
157 3323073 : SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
158 3323073 : GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
159 3323073 : if (TryCand.Reason != NoCand) {
160 : // Initialize resource delta if needed in case future heuristics query it.
161 1164983 : if (TryCand.ResDelta == SchedResourceDelta())
162 1164983 : TryCand.initResourceDelta(Zone.DAG, SchedModel);
163 : Cand.setBest(TryCand);
164 : }
165 : }
166 314616 : }
167 :
168 : // This function is mostly cut and pasted from
169 : // GenericScheduler::pickNodeBidirectional()
170 341436 : SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
171 : // Schedule as far as possible in the direction of no choice. This is most
172 : // efficient, but also provides the best heuristics for CriticalPSets.
173 341436 : if (SUnit *SU = Bot.pickOnlyChoice()) {
174 64167 : IsTopNode = false;
175 64167 : return SU;
176 : }
177 277269 : if (SUnit *SU = Top.pickOnlyChoice()) {
178 6902 : IsTopNode = true;
179 6902 : return SU;
180 : }
181 : // Set the bottom-up policy based on the state of the current bottom zone and
182 : // the instructions outside the zone, including the top zone.
183 270367 : CandPolicy BotPolicy;
184 270367 : setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
185 : // Set the top-down policy based on the state of the current top zone and
186 : // the instructions outside the zone, including the bottom zone.
187 270367 : CandPolicy TopPolicy;
188 270367 : setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
189 :
190 : // See if BotCand is still valid (because we previously scheduled from Top).
191 : LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
192 270367 : if (!BotCand.isValid() || BotCand.SU->isScheduled ||
193 : BotCand.Policy != BotPolicy) {
194 218227 : BotCand.reset(CandPolicy());
195 436454 : pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
196 : assert(BotCand.Reason != NoCand && "failed to find the first candidate");
197 : } else {
198 : LLVM_DEBUG(traceCandidate(BotCand));
199 : }
200 :
201 : // Check if the top Q has a better candidate.
202 : LLVM_DEBUG(dbgs() << "Picking from Top:\n");
203 270367 : if (!TopCand.isValid() || TopCand.SU->isScheduled ||
204 : TopCand.Policy != TopPolicy) {
205 96389 : TopCand.reset(CandPolicy());
206 192778 : pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
207 : assert(TopCand.Reason != NoCand && "failed to find the first candidate");
208 : } else {
209 : LLVM_DEBUG(traceCandidate(TopCand));
210 : }
211 :
212 : // Pick best from BotCand and TopCand.
213 : LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
214 : dbgs() << "Bot Cand: "; traceCandidate(BotCand););
215 : SchedCandidate Cand;
216 270367 : if (TopCand.Reason == BotCand.Reason) {
217 122747 : Cand = BotCand;
218 : GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
219 122747 : TopCand.Reason = NoCand;
220 122747 : GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
221 122747 : if (TopCand.Reason != NoCand) {
222 : Cand.setBest(TopCand);
223 : } else {
224 104468 : TopCand.Reason = TopReason;
225 : }
226 : } else {
227 147620 : if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
228 9 : Cand = TopCand;
229 147611 : } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
230 130 : Cand = BotCand;
231 147481 : } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
232 402 : Cand = TopCand;
233 147079 : } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
234 12151 : Cand = BotCand;
235 : } else {
236 134928 : if (BotCand.Reason > TopCand.Reason) {
237 44317 : Cand = TopCand;
238 : } else {
239 90611 : Cand = BotCand;
240 : }
241 : }
242 : }
243 : LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
244 :
245 270367 : IsTopNode = Cand.AtTop;
246 270367 : return Cand.SU;
247 : }
248 :
249 : // This function is mostly cut and pasted from
250 : // GenericScheduler::pickNode()
251 362888 : SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
252 362888 : if (DAG->top() == DAG->bottom()) {
253 : assert(Top.Available.empty() && Top.Pending.empty() &&
254 : Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
255 : return nullptr;
256 : }
257 : SUnit *SU;
258 : do {
259 341436 : if (RegionPolicy.OnlyTopDown) {
260 0 : SU = Top.pickOnlyChoice();
261 0 : if (!SU) {
262 0 : CandPolicy NoPolicy;
263 0 : TopCand.reset(NoPolicy);
264 0 : pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
265 : assert(TopCand.Reason != NoCand && "failed to find a candidate");
266 0 : SU = TopCand.SU;
267 : }
268 0 : IsTopNode = true;
269 341436 : } else if (RegionPolicy.OnlyBottomUp) {
270 0 : SU = Bot.pickOnlyChoice();
271 0 : if (!SU) {
272 0 : CandPolicy NoPolicy;
273 0 : BotCand.reset(NoPolicy);
274 0 : pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
275 : assert(BotCand.Reason != NoCand && "failed to find a candidate");
276 0 : SU = BotCand.SU;
277 : }
278 0 : IsTopNode = false;
279 : } else {
280 341436 : SU = pickNodeBidirectional(IsTopNode);
281 : }
282 341436 : } while (SU->isScheduled);
283 :
284 341436 : if (SU->isTopReady())
285 146623 : Top.removeReady(SU);
286 341436 : if (SU->isBottomReady())
287 294763 : Bot.removeReady(SU);
288 :
289 : LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
290 : << *SU->getInstr());
291 : return SU;
292 : }
293 :
294 19515 : GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
295 19515 : std::unique_ptr<MachineSchedStrategy> S) :
296 : ScheduleDAGMILive(C, std::move(S)),
297 19515 : ST(MF.getSubtarget<GCNSubtarget>()),
298 19515 : MFI(*MF.getInfo<SIMachineFunctionInfo>()),
299 19515 : StartingOccupancy(MFI.getOccupancy()),
300 58545 : MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
301 :
302 : LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
303 19515 : }
304 :
305 42556 : void GCNScheduleDAGMILive::schedule() {
306 42556 : if (Stage == 0) {
307 : // Just record regions at the first pass.
308 42216 : Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
309 42544 : return;
310 : }
311 :
312 : std::vector<MachineInstr*> Unsched;
313 21448 : Unsched.reserve(NumRegionInstrs);
314 361694 : for (auto &I : *this) {
315 340246 : Unsched.push_back(&I);
316 : }
317 :
318 : GCNRegPressure PressureBefore;
319 21448 : if (LIS) {
320 42896 : PressureBefore = Pressure[RegionIdx];
321 :
322 : LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
323 : GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
324 : dbgs() << "Region live-in pressure: ";
325 : llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
326 : dbgs() << "Region register pressure: ";
327 : PressureBefore.print(dbgs()));
328 : }
329 :
330 21448 : ScheduleDAGMILive::schedule();
331 21448 : Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
332 :
333 21448 : if (!LIS)
334 : return;
335 :
336 : // Check the results of scheduling.
337 : GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
338 21448 : auto PressureAfter = getRealRegPressure();
339 :
340 : LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";
341 : PressureAfter.print(dbgs()));
342 :
343 21448 : if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
344 21331 : PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
345 20836 : Pressure[RegionIdx] = PressureAfter;
346 : LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
347 20836 : return;
348 : }
349 612 : unsigned Occ = MFI.getOccupancy();
350 612 : unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
351 875 : unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
352 : LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
353 : << ", after " << WavesAfter << ".\n");
354 :
355 : // We could not keep current target occupancy because of the just scheduled
356 : // region. Record new occupancy for next scheduling cycle.
357 612 : unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
358 : // Allow memory bound functions to drop to 4 waves if not limited by an
359 : // attribute.
360 648 : if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
361 36 : WavesAfter >= MFI.getMinAllowedOccupancy()) {
362 : LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "
363 : << MFI.getMinAllowedOccupancy() << " waves\n");
364 : NewOccupancy = WavesAfter;
365 : }
366 612 : if (NewOccupancy < MinOccupancy) {
367 252 : MinOccupancy = NewOccupancy;
368 252 : MFI.limitOccupancy(MinOccupancy);
369 : LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
370 : << MinOccupancy << ".\n");
371 : }
372 :
373 612 : if (WavesAfter >= MinOccupancy) {
374 600 : Pressure[RegionIdx] = PressureAfter;
375 600 : return;
376 : }
377 :
378 : LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n");
379 12 : RegionEnd = RegionBegin;
380 1216 : for (MachineInstr *MI : Unsched) {
381 : if (MI->isDebugInstr())
382 0 : continue;
383 :
384 2408 : if (MI->getIterator() != RegionEnd) {
385 665 : BB->remove(MI);
386 665 : BB->insert(RegionEnd, MI);
387 : if (!MI->isDebugInstr())
388 665 : LIS->handleMove(*MI, true);
389 : }
390 : // Reset read-undef flags and update them later.
391 7318 : for (auto &Op : MI->operands())
392 6114 : if (Op.isReg() && Op.isDef())
393 : Op.setIsUndef(false);
394 1204 : RegisterOperands RegOpers;
395 1204 : RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
396 : if (!MI->isDebugInstr()) {
397 1204 : if (ShouldTrackLaneMasks) {
398 : // Adjust liveness and add missing dead+read-undef flags.
399 1204 : SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
400 1204 : RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
401 : } else {
402 : // Adjust for missing dead-def flags.
403 0 : RegOpers.detectDeadDefs(*MI, *LIS);
404 : }
405 : }
406 1204 : RegionEnd = MI->getIterator();
407 : ++RegionEnd;
408 : LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
409 : }
410 12 : RegionBegin = Unsched.front()->getIterator();
411 12 : Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
412 :
413 12 : placeDebugValues();
414 : }
415 :
416 21448 : GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
417 21448 : GCNDownwardRPTracker RPTracker(*LIS);
418 42896 : RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
419 21448 : return RPTracker.moveMaxPressure();
420 : }
421 :
422 20403 : void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
423 20403 : GCNDownwardRPTracker RPTracker(*LIS);
424 :
425 : // If the block has the only successor then live-ins of that successor are
426 : // live-outs of the current block. We can reuse calculated live set if the
427 : // successor will be sent to scheduling past current block.
428 20403 : const MachineBasicBlock *OnlySucc = nullptr;
429 20403 : if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
430 950 : SlotIndexes *Ind = LIS->getSlotIndexes();
431 950 : if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
432 875 : OnlySucc = *MBB->succ_begin();
433 : }
434 :
435 : // Scheduler sends regions from the end of the block upwards.
436 20403 : size_t CurRegion = RegionIdx;
437 41511 : for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
438 22688 : if (Regions[CurRegion].first->getParent() != MBB)
439 : break;
440 20403 : --CurRegion;
441 :
442 : auto I = MBB->begin();
443 20403 : auto LiveInIt = MBBLiveIns.find(MBB);
444 20403 : if (LiveInIt != MBBLiveIns.end()) {
445 : auto LiveIn = std::move(LiveInIt->second);
446 570 : RPTracker.reset(*MBB->begin(), &LiveIn);
447 : MBBLiveIns.erase(LiveInIt);
448 : } else {
449 : I = Regions[CurRegion].first;
450 19833 : RPTracker.reset(*I);
451 : }
452 :
453 : for ( ; ; ) {
454 312607 : I = RPTracker.getNext();
455 :
456 333010 : if (Regions[CurRegion].first == I) {
457 : LiveIns[CurRegion] = RPTracker.getLiveRegs();
458 : RPTracker.clearMaxPressure();
459 : }
460 :
461 333010 : if (Regions[CurRegion].second == I) {
462 21108 : Pressure[CurRegion] = RPTracker.moveMaxPressure();
463 21108 : if (CurRegion-- == RegionIdx)
464 : break;
465 : }
466 312607 : RPTracker.advanceToNext();
467 312607 : RPTracker.advanceBeforeNext();
468 : }
469 :
470 20403 : if (OnlySucc) {
471 875 : if (I != MBB->end()) {
472 173 : RPTracker.advanceToNext();
473 173 : RPTracker.advance(MBB->end());
474 : }
475 1750 : RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
476 875 : RPTracker.advanceBeforeNext();
477 1750 : MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
478 : }
479 20403 : }
480 :
481 19515 : void GCNScheduleDAGMILive::finalizeSchedule() {
482 : GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
483 : LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
484 :
485 39030 : LiveIns.resize(Regions.size());
486 39030 : Pressure.resize(Regions.size());
487 :
488 : do {
489 39030 : Stage++;
490 39030 : RegionIdx = 0;
491 : MachineBasicBlock *MBB = nullptr;
492 :
493 39030 : if (Stage > 1) {
494 : // Retry function scheduling if we found resulting occupancy and it is
495 : // lower than used for first pass scheduling. This will give more freedom
496 : // to schedule low register pressure blocks.
497 : // Code is partially copied from MachineSchedulerBase::scheduleRegions().
498 :
499 19515 : if (!LIS || StartingOccupancy <= MinOccupancy)
500 : break;
501 :
502 : LLVM_DEBUG(
503 : dbgs()
504 : << "Retrying function scheduling with lowest recorded occupancy "
505 : << MinOccupancy << ".\n");
506 :
507 : S.setTargetOccupancy(MinOccupancy);
508 : }
509 :
510 41213 : for (auto Region : Regions) {
511 21448 : RegionBegin = Region.first;
512 21448 : RegionEnd = Region.second;
513 :
514 21448 : if (RegionBegin->getParent() != MBB) {
515 20722 : if (MBB) finishBlock();
516 20722 : MBB = RegionBegin->getParent();
517 20722 : startBlock(MBB);
518 20722 : if (Stage == 1)
519 20403 : computeBlockPressure(MBB);
520 : }
521 :
522 21448 : unsigned NumRegionInstrs = std::distance(begin(), end());
523 21448 : enterRegion(MBB, begin(), end(), NumRegionInstrs);
524 :
525 : // Skip empty scheduling regions (0 or 1 schedulable instructions).
526 42896 : if (begin() == end() || begin() == std::prev(end())) {
527 0 : exitRegion();
528 0 : continue;
529 : }
530 :
531 : LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
532 : LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "
533 : << MBB->getName() << "\n From: " << *begin()
534 : << " To: ";
535 : if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
536 : else dbgs() << "End";
537 : dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
538 :
539 21448 : schedule();
540 :
541 21448 : exitRegion();
542 21448 : ++RegionIdx;
543 : }
544 19765 : finishBlock();
545 :
546 19765 : } while (Stage < 2);
547 19515 : }
|
