Bug Summary

File:lib/Target/AMDGPU/GCNSchedStrategy.cpp
Warning:line 534, column 7
Called C++ object pointer is null

Annotated Source Code

1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This contains a MachineSchedStrategy implementation for maximizing wave
12/// occupancy on GCN hardware.
13//===----------------------------------------------------------------------===//
14
15#include "GCNSchedStrategy.h"
16#include "AMDGPUSubtarget.h"
17#include "SIInstrInfo.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIRegisterInfo.h"
20#include "llvm/CodeGen/RegisterClassInfo.h"
21#include "llvm/Support/MathExtras.h"
22
23#define DEBUG_TYPE"misched" "misched"
24
25using namespace llvm;
26
27GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
28 const MachineSchedContext *C) :
29 GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
30
31static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs,
32 const MachineFunction &MF) {
33
34 const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
35 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
36 unsigned MinRegOccupancy = std::min(ST.getOccupancyWithNumSGPRs(SGPRs),
37 ST.getOccupancyWithNumVGPRs(VGPRs));
38 return std::min(MinRegOccupancy,
39 ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
40 *MF.getFunction()));
41}
42
43void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
44 GenericScheduler::initialize(DAG);
45
46 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
47
48 MF = &DAG->MF;
49
50 const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
51
52 // FIXME: This is also necessary, because some passes that run after
53 // scheduling and before regalloc increase register pressure.
54 const int ErrorMargin = 3;
55
56 SGPRExcessLimit = Context->RegClassInfo
57 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
58 VGPRExcessLimit = Context->RegClassInfo
59 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
60 if (TargetOccupancy) {
61 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
62 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
63 } else {
64 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
65 SRI->getSGPRPressureSet());
66 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
67 SRI->getVGPRPressureSet());
68 }
69
70 SGPRCriticalLimit -= ErrorMargin;
71 VGPRCriticalLimit -= ErrorMargin;
72}
73
74void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
75 bool AtTop, const RegPressureTracker &RPTracker,
76 const SIRegisterInfo *SRI,
77 unsigned SGPRPressure,
78 unsigned VGPRPressure) {
79
80 Cand.SU = SU;
81 Cand.AtTop = AtTop;
82
83 // getDownwardPressure() and getUpwardPressure() make temporary changes to
84 // the the tracker, so we need to pass those function a non-const copy.
85 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
86
87 std::vector<unsigned> Pressure;
88 std::vector<unsigned> MaxPressure;
89
90 if (AtTop)
91 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
92 else {
93 // FIXME: I think for bottom up scheduling, the register pressure is cached
94 // and can be retrieved by DAG->getPressureDif(SU).
95 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
96 }
97
98 unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
99 unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
100
101 // If two instructions increase the pressure of different register sets
102 // by the same amount, the generic scheduler will prefer to schedule the
103 // instruction that increases the set with the least amount of registers,
104 // which in our case would be SGPRs. This is rarely what we want, so
105 // when we report excess/critical register pressure, we do it either
106 // only for VGPRs or only for SGPRs.
107
108 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
109 const unsigned MaxVGPRPressureInc = 16;
110 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
111 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
112
113
114 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
115 // to increase the likelihood we don't go over the limits. We should improve
116 // the analysis to look through dependencies to find the path with the least
117 // register pressure.
118
119 // We only need to update the RPDelata for instructions that increase
120 // register pressure. Instructions that decrease or keep reg pressure
121 // the same will be marked as RegExcess in tryCandidate() when they
122 // are compared with instructions that increase the register pressure.
123 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
124 Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
125 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
126 }
127
128 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
129 Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
130 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
131 }
132
133 // Register pressure is considered 'CRITICAL' if it is approaching a value
134 // that would reduce the wave occupancy for the execution unit. When
135 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
136 // has the same cost, so we don't need to prefer one over the other.
137
138 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
139 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
140
141 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
142 if (SGPRDelta > VGPRDelta) {
143 Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
144 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
145 } else {
146 Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
147 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
148 }
149 }
150}
151
152// This function is mostly cut and pasted from
153// GenericScheduler::pickNodeFromQueue()
154void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
155 const CandPolicy &ZonePolicy,
156 const RegPressureTracker &RPTracker,
157 SchedCandidate &Cand) {
158 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
159 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
160 unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
161 unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
162 ReadyQueue &Q = Zone.Available;
163 for (SUnit *SU : Q) {
164
165 SchedCandidate TryCand(ZonePolicy);
166 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
167 SGPRPressure, VGPRPressure);
168 // Pass SchedBoundary only when comparing nodes from the same boundary.
169 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
170 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
171 if (TryCand.Reason != NoCand) {
172 // Initialize resource delta if needed in case future heuristics query it.
173 if (TryCand.ResDelta == SchedResourceDelta())
174 TryCand.initResourceDelta(Zone.DAG, SchedModel);
175 Cand.setBest(TryCand);
176 }
177 }
178}
179
180// This function is mostly cut and pasted from
181// GenericScheduler::pickNodeBidirectional()
182SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
183 // Schedule as far as possible in the direction of no choice. This is most
184 // efficient, but also provides the best heuristics for CriticalPSets.
185 if (SUnit *SU = Bot.pickOnlyChoice()) {
186 IsTopNode = false;
187 return SU;
188 }
189 if (SUnit *SU = Top.pickOnlyChoice()) {
190 IsTopNode = true;
191 return SU;
192 }
193 // Set the bottom-up policy based on the state of the current bottom zone and
194 // the instructions outside the zone, including the top zone.
195 CandPolicy BotPolicy;
196 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
197 // Set the top-down policy based on the state of the current top zone and
198 // the instructions outside the zone, including the bottom zone.
199 CandPolicy TopPolicy;
200 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
201
202 // See if BotCand is still valid (because we previously scheduled from Top).
203 DEBUG(dbgs() << "Picking from Bot:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking from Bot:\n"; } } while
(false)
;
204 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
205 BotCand.Policy != BotPolicy) {
206 BotCand.reset(CandPolicy());
207 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
208 assert(BotCand.Reason != NoCand && "failed to find the first candidate")((BotCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find the first candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 208, __PRETTY_FUNCTION__))
;
209 } else {
210 DEBUG(traceCandidate(BotCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { traceCandidate(BotCand); } } while (false)
;
211 }
212
213 // Check if the top Q has a better candidate.
214 DEBUG(dbgs() << "Picking from Top:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking from Top:\n"; } } while
(false)
;
215 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
216 TopCand.Policy != TopPolicy) {
217 TopCand.reset(CandPolicy());
218 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
219 assert(TopCand.Reason != NoCand && "failed to find the first candidate")((TopCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find the first candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 219, __PRETTY_FUNCTION__))
;
220 } else {
221 DEBUG(traceCandidate(TopCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { traceCandidate(TopCand); } } while (false)
;
222 }
223
224 // Pick best from BotCand and TopCand.
225 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
226 dbgs() << "Top Cand: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
227 traceCandidate(TopCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
228 dbgs() << "Bot Cand: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
229 traceCandidate(BotCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
230 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
;
231 SchedCandidate Cand;
232 if (TopCand.Reason == BotCand.Reason) {
233 Cand = BotCand;
234 GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
235 TopCand.Reason = NoCand;
236 GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
237 if (TopCand.Reason != NoCand) {
238 Cand.setBest(TopCand);
239 } else {
240 TopCand.Reason = TopReason;
241 }
242 } else {
243 if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
244 Cand = TopCand;
245 } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
246 Cand = BotCand;
247 } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
248 Cand = TopCand;
249 } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
250 Cand = BotCand;
251 } else {
252 if (BotCand.Reason > TopCand.Reason) {
253 Cand = TopCand;
254 } else {
255 Cand = BotCand;
256 }
257 }
258 }
259 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
260 dbgs() << "Picking: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
261 traceCandidate(Cand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
262 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
;
263
264 IsTopNode = Cand.AtTop;
265 return Cand.SU;
266}
267
268// This function is mostly cut and pasted from
269// GenericScheduler::pickNode()
270SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
271 if (DAG->top() == DAG->bottom()) {
272 assert(Top.Available.empty() && Top.Pending.empty() &&((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 273, __PRETTY_FUNCTION__))
273 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage")((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 273, __PRETTY_FUNCTION__))
;
274 return nullptr;
275 }
276 SUnit *SU;
277 do {
278 if (RegionPolicy.OnlyTopDown) {
279 SU = Top.pickOnlyChoice();
280 if (!SU) {
281 CandPolicy NoPolicy;
282 TopCand.reset(NoPolicy);
283 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
284 assert(TopCand.Reason != NoCand && "failed to find a candidate")((TopCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find a candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 284, __PRETTY_FUNCTION__))
;
285 SU = TopCand.SU;
286 }
287 IsTopNode = true;
288 } else if (RegionPolicy.OnlyBottomUp) {
289 SU = Bot.pickOnlyChoice();
290 if (!SU) {
291 CandPolicy NoPolicy;
292 BotCand.reset(NoPolicy);
293 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
294 assert(BotCand.Reason != NoCand && "failed to find a candidate")((BotCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find a candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 294, __PRETTY_FUNCTION__))
;
295 SU = BotCand.SU;
296 }
297 IsTopNode = false;
298 } else {
299 SU = pickNodeBidirectional(IsTopNode);
300 }
301 } while (SU->isScheduled);
302
303 if (SU->isTopReady())
304 Top.removeReady(SU);
305 if (SU->isBottomReady())
306 Bot.removeReady(SU);
307
308 DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Scheduling SU(" << SU->
NodeNum << ") " << *SU->getInstr(); } } while (
false)
;
309 return SU;
310}
311
312GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
313 std::unique_ptr<MachineSchedStrategy> S) :
314 ScheduleDAGMILive(C, std::move(S)),
315 ST(MF.getSubtarget<SISubtarget>()),
316 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
317 StartingOccupancy(ST.getOccupancyWithLocalMemSize(MFI.getLDSSize(),
318 *MF.getFunction())),
319 MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
320
321 DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Starting occupancy is " <<
StartingOccupancy << ".\n"; } } while (false)
;
322}
323
324void GCNScheduleDAGMILive::schedule() {
325 if (Stage == 0) {
326 // Just record regions at the first pass.
327 Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
328 return;
329 }
330
331 std::vector<MachineInstr*> Unsched;
332 Unsched.reserve(NumRegionInstrs);
333 for (auto &I : *this)
334 Unsched.push_back(&I);
335
336 GCNRegPressure PressureBefore;
337 if (LIS) {
338 PressureBefore = Pressure[RegionIdx];
339
340 DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
341 GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
342 dbgs() << "Region live-in pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
343 llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
344 dbgs() << "Region register pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
345 PressureBefore.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
;
346 }
347
348 ScheduleDAGMILive::schedule();
349 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
350
351 if (!LIS)
352 return;
353
354 // Check the results of scheduling.
355 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
356 auto PressureAfter = getRealRegPressure();
357
358 DEBUG(dbgs() << "Pressure after scheduling: "; PressureAfter.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure after scheduling: ";
PressureAfter.print(dbgs()); } } while (false)
;
359
360 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
361 PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
362 Pressure[RegionIdx] = PressureAfter;
363 DEBUG(dbgs() << "Pressure in desired limits, done.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure in desired limits, done.\n"
; } } while (false)
;
364 return;
365 }
366 unsigned WavesAfter = getMaxWaves(PressureAfter.getSGPRNum(),
367 PressureAfter.getVGPRNum(), MF);
368 unsigned WavesBefore = getMaxWaves(PressureBefore.getSGPRNum(),
369 PressureBefore.getVGPRNum(), MF);
370 DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
371 ", after " << WavesAfter << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
;
372
373 // We could not keep current target occupancy because of the just scheduled
374 // region. Record new occupancy for next scheduling cycle.
375 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
376 if (NewOccupancy < MinOccupancy) {
377 MinOccupancy = NewOccupancy;
378 DEBUG(dbgs() << "Occupancy lowered for the function to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
379 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
;
380 }
381
382 if (WavesAfter >= WavesBefore) {
383 Pressure[RegionIdx] = PressureAfter;
384 return;
385 }
386
387 DEBUG(dbgs() << "Attempting to revert scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Attempting to revert scheduling.\n"
; } } while (false)
;
388 RegionEnd = RegionBegin;
389 for (MachineInstr *MI : Unsched) {
390 if (MI->getIterator() != RegionEnd) {
391 BB->remove(MI);
392 BB->insert(RegionEnd, MI);
393 LIS->handleMove(*MI, true);
394 }
395 // Reset read-undef flags and update them later.
396 for (auto &Op : MI->operands())
397 if (Op.isReg() && Op.isDef())
398 Op.setIsUndef(false);
399 RegisterOperands RegOpers;
400 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
401 if (ShouldTrackLaneMasks) {
402 // Adjust liveness and add missing dead+read-undef flags.
403 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
404 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
405 } else {
406 // Adjust for missing dead-def flags.
407 RegOpers.detectDeadDefs(*MI, *LIS);
408 }
409 RegionEnd = MI->getIterator();
410 ++RegionEnd;
411 DEBUG(dbgs() << "Scheduling " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Scheduling " << *MI; } }
while (false)
;
412 }
413 RegionBegin = Unsched.front()->getIterator();
414 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
415
416 placeDebugValues();
417}
418
419GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
420 GCNDownwardRPTracker RPTracker(*LIS);
421 RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
422 return RPTracker.moveMaxPressure();
423}
424
425void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
426 GCNDownwardRPTracker RPTracker(*LIS);
427
428 // If the block has the only successor then live-ins of that successor are
429 // live-outs of the current block. We can reuse calculated live set if the
430 // successor will be sent to scheduling past current block.
431 const MachineBasicBlock *OnlySucc = nullptr;
432 if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
433 SlotIndexes *Ind = LIS->getSlotIndexes();
434 if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
435 OnlySucc = *MBB->succ_begin();
436 }
437
438 // Scheduler sends regions from the end of the block upwards.
439 size_t CurRegion = RegionIdx;
440 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
441 if (Regions[CurRegion].first->getParent() != MBB)
442 break;
443 --CurRegion;
444
445 auto I = MBB->begin();
446 auto LiveInIt = MBBLiveIns.find(MBB);
447 if (LiveInIt != MBBLiveIns.end()) {
448 auto LiveIn = std::move(LiveInIt->second);
449 RPTracker.reset(*MBB->begin(), &LiveIn);
450 MBBLiveIns.erase(LiveInIt);
451 } else {
452 I = Regions[CurRegion].first;
453 RPTracker.reset(*I);
454 }
455
456 for ( ; ; ) {
457 I = RPTracker.getNext();
458
459 if (Regions[CurRegion].first == I) {
460 LiveIns[CurRegion] = RPTracker.getLiveRegs();
461 RPTracker.clearMaxPressure();
462 }
463
464 if (Regions[CurRegion].second == I) {
465 Pressure[CurRegion] = RPTracker.moveMaxPressure();
466 if (CurRegion-- == RegionIdx)
467 break;
468 }
469 RPTracker.advanceToNext();
470 RPTracker.advanceBeforeNext();
471 }
472
473 if (OnlySucc) {
474 if (I != MBB->end()) {
475 RPTracker.advanceToNext();
476 RPTracker.advance(MBB->end());
477 }
478 RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
479 RPTracker.advanceBeforeNext();
480 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
481 }
482}
483
484void GCNScheduleDAGMILive::finalizeSchedule() {
485 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
486 DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "All regions recorded, starting actual scheduling.\n"
; } } while (false)
;
487
488 LiveIns.resize(Regions.size());
489 Pressure.resize(Regions.size());
490
491 do {
5
Loop condition is true. Execution continues on line 492
10
Loop condition is true. Execution continues on line 492
15
Loop condition is true. Execution continues on line 492
492 Stage++;
493 RegionIdx = 0;
494 MachineBasicBlock *MBB = nullptr;
16
'MBB' initialized to a null pointer value
495
496 if (Stage > 1) {
1
Assuming the condition is false
2
Taking false branch
6
Assuming the condition is false
7
Taking false branch
11
Assuming the condition is false
12
Taking false branch
17
Assuming the condition is false
18
Taking false branch
497 // Retry function scheduling if we found resulting occupancy and it is
498 // lower than used for first pass scheduling. This will give more freedom
499 // to schedule low register pressure blocks.
500 // Code is partially copied from MachineSchedulerBase::scheduleRegions().
501
502 if (!LIS || StartingOccupancy <= MinOccupancy)
503 break;
504
505 DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
506 << "Retrying function scheduling with lowest recorded occupancy "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
507 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
;
508
509 S.setTargetOccupancy(MinOccupancy);
510 }
511
512 for (auto Region : Regions) {
3
Assuming '__begin' is equal to '__end'
8
Assuming '__begin' is equal to '__end'
13
Assuming '__begin' is equal to '__end'
19
Assuming '__begin' is not equal to '__end'
513 RegionBegin = Region.first;
514 RegionEnd = Region.second;
515
516 if (RegionBegin->getParent() != MBB) {
20
Assuming the condition is false
21
Taking false branch
517 if (MBB) finishBlock();
518 MBB = RegionBegin->getParent();
519 startBlock(MBB);
520 if (Stage == 1)
521 computeBlockPressure(MBB);
522 }
523
524 unsigned NumRegionInstrs = std::distance(begin(), end());
525 enterRegion(MBB, begin(), end(), NumRegionInstrs);
526
527 // Skip empty scheduling regions (0 or 1 schedulable instructions).
528 if (begin() == end() || begin() == std::prev(end())) {
22
Taking false branch
529 exitRegion();
530 continue;
531 }
532
533 DEBUG(dbgs() << "********** MI Scheduling **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "********** MI Scheduling **********\n"
; } } while (false)
;
534 DEBUG(dbgs() << MF.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
23
Within the expansion of the macro 'DEBUG':
a
Assuming 'DebugFlag' is not equal to 0
b
Assuming the condition is true
c
Called C++ object pointer is null
535 << ":BB#" << MBB->getNumber() << " " << MBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
536 << "\n From: " << *begin() << " To: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
537 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
538 else dbgs() << "End";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
539 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
;
540
541 schedule();
542
543 exitRegion();
544 ++RegionIdx;
545 }
546 finishBlock();
547
548 } while (Stage < 2);
4
Assuming the condition is true
9
Assuming the condition is true
14
Assuming the condition is true
549}