Bug Summary

File:lib/Target/AMDGPU/GCNSchedStrategy.cpp
Warning:line 556, column 5
Called C++ object pointer is null

Annotated Source Code

1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This contains a MachineSchedStrategy implementation for maximizing wave
12/// occupancy on GCN hardware.
13//===----------------------------------------------------------------------===//
14
15#include "GCNSchedStrategy.h"
16#include "AMDGPUSubtarget.h"
17#include "SIInstrInfo.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIRegisterInfo.h"
20#include "llvm/CodeGen/RegisterClassInfo.h"
21#include "llvm/Support/MathExtras.h"
22
23#define DEBUG_TYPE"misched" "misched"
24
25using namespace llvm;
26
27GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
28 const MachineSchedContext *C) :
29 GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
30
31static unsigned getMaxWaves(unsigned SGPRs, unsigned VGPRs,
32 const MachineFunction &MF) {
33
34 const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
35 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
36 unsigned MinRegOccupancy = std::min(ST.getOccupancyWithNumSGPRs(SGPRs),
37 ST.getOccupancyWithNumVGPRs(VGPRs));
38 return std::min(MinRegOccupancy,
39 ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
40 *MF.getFunction()));
41}
42
43void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
44 GenericScheduler::initialize(DAG);
45
46 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
47
48 if (MF != &DAG->MF)
49 TargetOccupancy = 0;
50 MF = &DAG->MF;
51
52 const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
53
54 // FIXME: This is also necessary, because some passes that run after
55 // scheduling and before regalloc increase register pressure.
56 const int ErrorMargin = 3;
57
58 SGPRExcessLimit = Context->RegClassInfo
59 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
60 VGPRExcessLimit = Context->RegClassInfo
61 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
62 if (TargetOccupancy) {
63 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
64 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
65 } else {
66 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
67 SRI->getSGPRPressureSet());
68 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
69 SRI->getVGPRPressureSet());
70 }
71
72 SGPRCriticalLimit -= ErrorMargin;
73 VGPRCriticalLimit -= ErrorMargin;
74}
75
76void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
77 bool AtTop, const RegPressureTracker &RPTracker,
78 const SIRegisterInfo *SRI,
79 unsigned SGPRPressure,
80 unsigned VGPRPressure) {
81
82 Cand.SU = SU;
83 Cand.AtTop = AtTop;
84
85 // getDownwardPressure() and getUpwardPressure() make temporary changes to
86 // the the tracker, so we need to pass those function a non-const copy.
87 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
88
89 std::vector<unsigned> Pressure;
90 std::vector<unsigned> MaxPressure;
91
92 if (AtTop)
93 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
94 else {
95 // FIXME: I think for bottom up scheduling, the register pressure is cached
96 // and can be retrieved by DAG->getPressureDif(SU).
97 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
98 }
99
100 unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
101 unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
102
103 // If two instructions increase the pressure of different register sets
104 // by the same amount, the generic scheduler will prefer to schedule the
105 // instruction that increases the set with the least amount of registers,
106 // which in our case would be SGPRs. This is rarely what we want, so
107 // when we report excess/critical register pressure, we do it either
108 // only for VGPRs or only for SGPRs.
109
110 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
111 const unsigned MaxVGPRPressureInc = 16;
112 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
113 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
114
115
116 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
117 // to increase the likelihood we don't go over the limits. We should improve
118 // the analysis to look through dependencies to find the path with the least
119 // register pressure.
120
121 // We only need to update the RPDelata for instructions that increase
122 // register pressure. Instructions that decrease or keep reg pressure
123 // the same will be marked as RegExcess in tryCandidate() when they
124 // are compared with instructions that increase the register pressure.
125 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
126 Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
127 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
128 }
129
130 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
131 Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
132 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
133 }
134
135 // Register pressure is considered 'CRITICAL' if it is approaching a value
136 // that would reduce the wave occupancy for the execution unit. When
137 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
138 // has the same cost, so we don't need to prefer one over the other.
139
140 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
141 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
142
143 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
144 if (SGPRDelta > VGPRDelta) {
145 Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
146 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
147 } else {
148 Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
149 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
150 }
151 }
152}
153
154// This function is mostly cut and pasted from
155// GenericScheduler::pickNodeFromQueue()
156void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
157 const CandPolicy &ZonePolicy,
158 const RegPressureTracker &RPTracker,
159 SchedCandidate &Cand) {
160 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
161 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
162 unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
163 unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
164 ReadyQueue &Q = Zone.Available;
165 for (SUnit *SU : Q) {
166
167 SchedCandidate TryCand(ZonePolicy);
168 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
169 SGPRPressure, VGPRPressure);
170 // Pass SchedBoundary only when comparing nodes from the same boundary.
171 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
172 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
173 if (TryCand.Reason != NoCand) {
174 // Initialize resource delta if needed in case future heuristics query it.
175 if (TryCand.ResDelta == SchedResourceDelta())
176 TryCand.initResourceDelta(Zone.DAG, SchedModel);
177 Cand.setBest(TryCand);
178 }
179 }
180}
181
182// This function is mostly cut and pasted from
183// GenericScheduler::pickNodeBidirectional()
184SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
185 // Schedule as far as possible in the direction of no choice. This is most
186 // efficient, but also provides the best heuristics for CriticalPSets.
187 if (SUnit *SU = Bot.pickOnlyChoice()) {
188 IsTopNode = false;
189 return SU;
190 }
191 if (SUnit *SU = Top.pickOnlyChoice()) {
192 IsTopNode = true;
193 return SU;
194 }
195 // Set the bottom-up policy based on the state of the current bottom zone and
196 // the instructions outside the zone, including the top zone.
197 CandPolicy BotPolicy;
198 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
199 // Set the top-down policy based on the state of the current top zone and
200 // the instructions outside the zone, including the bottom zone.
201 CandPolicy TopPolicy;
202 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
203
204 // See if BotCand is still valid (because we previously scheduled from Top).
205 DEBUG(dbgs() << "Picking from Bot:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking from Bot:\n"; } } while
(false)
;
206 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
207 BotCand.Policy != BotPolicy) {
208 BotCand.reset(CandPolicy());
209 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
210 assert(BotCand.Reason != NoCand && "failed to find the first candidate")((BotCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find the first candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 210, __PRETTY_FUNCTION__))
;
211 } else {
212 DEBUG(traceCandidate(BotCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { traceCandidate(BotCand); } } while (false)
;
213 }
214
215 // Check if the top Q has a better candidate.
216 DEBUG(dbgs() << "Picking from Top:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking from Top:\n"; } } while
(false)
;
217 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
218 TopCand.Policy != TopPolicy) {
219 TopCand.reset(CandPolicy());
220 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
221 assert(TopCand.Reason != NoCand && "failed to find the first candidate")((TopCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find the first candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 221, __PRETTY_FUNCTION__))
;
222 } else {
223 DEBUG(traceCandidate(TopCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { traceCandidate(TopCand); } } while (false)
;
224 }
225
226 // Pick best from BotCand and TopCand.
227 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
228 dbgs() << "Top Cand: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
229 traceCandidate(TopCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
230 dbgs() << "Bot Cand: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
231 traceCandidate(BotCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
232 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Top Cand: "; traceCandidate(TopCand
); dbgs() << "Bot Cand: "; traceCandidate(BotCand);; } }
while (false)
;
233 SchedCandidate Cand;
234 if (TopCand.Reason == BotCand.Reason) {
235 Cand = BotCand;
236 GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
237 TopCand.Reason = NoCand;
238 GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
239 if (TopCand.Reason != NoCand) {
240 Cand.setBest(TopCand);
241 } else {
242 TopCand.Reason = TopReason;
243 }
244 } else {
245 if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
246 Cand = TopCand;
247 } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
248 Cand = BotCand;
249 } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
250 Cand = TopCand;
251 } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
252 Cand = BotCand;
253 } else {
254 if (BotCand.Reason > TopCand.Reason) {
255 Cand = TopCand;
256 } else {
257 Cand = BotCand;
258 }
259 }
260 }
261 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
262 dbgs() << "Picking: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
263 traceCandidate(Cand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
264 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Picking: "; traceCandidate(Cand
);; } } while (false)
;
265
266 IsTopNode = Cand.AtTop;
267 return Cand.SU;
268}
269
270// This function is mostly cut and pasted from
271// GenericScheduler::pickNode()
272SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
273 if (DAG->top() == DAG->bottom()) {
274 assert(Top.Available.empty() && Top.Pending.empty() &&((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 275, __PRETTY_FUNCTION__))
275 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage")((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 275, __PRETTY_FUNCTION__))
;
276 return nullptr;
277 }
278 SUnit *SU;
279 do {
280 if (RegionPolicy.OnlyTopDown) {
281 SU = Top.pickOnlyChoice();
282 if (!SU) {
283 CandPolicy NoPolicy;
284 TopCand.reset(NoPolicy);
285 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
286 assert(TopCand.Reason != NoCand && "failed to find a candidate")((TopCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find a candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 286, __PRETTY_FUNCTION__))
;
287 SU = TopCand.SU;
288 }
289 IsTopNode = true;
290 } else if (RegionPolicy.OnlyBottomUp) {
291 SU = Bot.pickOnlyChoice();
292 if (!SU) {
293 CandPolicy NoPolicy;
294 BotCand.reset(NoPolicy);
295 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
296 assert(BotCand.Reason != NoCand && "failed to find a candidate")((BotCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find a candidate\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 296, __PRETTY_FUNCTION__))
;
297 SU = BotCand.SU;
298 }
299 IsTopNode = false;
300 } else {
301 SU = pickNodeBidirectional(IsTopNode);
302 }
303 } while (SU->isScheduled);
304
305 if (SU->isTopReady())
306 Top.removeReady(SU);
307 if (SU->isBottomReady())
308 Bot.removeReady(SU);
309
310 DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Scheduling SU(" << SU->
NodeNum << ") " << *SU->getInstr(); } } while (
false)
;
311 return SU;
312}
313
314GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
315 std::unique_ptr<MachineSchedStrategy> S) :
316 ScheduleDAGMILive(C, std::move(S)),
317 ST(MF.getSubtarget<SISubtarget>()),
318 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
319 StartingOccupancy(ST.getOccupancyWithLocalMemSize(MFI.getLDSSize(),
320 *MF.getFunction())),
321 MinOccupancy(StartingOccupancy), Stage(0) {
322
323 DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Starting occupancy is " <<
StartingOccupancy << ".\n"; } } while (false)
;
324}
325
326void GCNScheduleDAGMILive::enterRegion(MachineBasicBlock *bb,
327 MachineBasicBlock::iterator begin,
328 MachineBasicBlock::iterator end,
329 unsigned regioninstrs) {
330 ScheduleDAGMILive::enterRegion(bb, begin, end, regioninstrs);
331
332 if (Stage == 0)
333 Regions.push_back(std::make_pair(begin, end));
334}
335
336void GCNScheduleDAGMILive::schedule() {
337 std::vector<MachineInstr*> Unsched;
338 Unsched.reserve(NumRegionInstrs);
339 for (auto &I : *this)
340 Unsched.push_back(&I);
341
342 std::pair<unsigned, unsigned> PressureBefore;
343 if (LIS) {
344 DEBUG(dbgs() << "Pressure before scheduling:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure before scheduling:\n"
; } } while (false)
;
345 discoverLiveIns();
346 PressureBefore = getRealRegPressure();
347 }
348
349 ScheduleDAGMILive::schedule();
350 if (!LIS)
351 return;
352
353 // Check the results of scheduling.
354 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
355 DEBUG(dbgs() << "Pressure after scheduling:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure after scheduling:\n"
; } } while (false)
;
356 auto PressureAfter = getRealRegPressure();
357 LiveIns.clear();
358
359 if (PressureAfter.first <= S.SGPRCriticalLimit &&
360 PressureAfter.second <= S.VGPRCriticalLimit) {
361 DEBUG(dbgs() << "Pressure in desired limits, done.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Pressure in desired limits, done.\n"
; } } while (false)
;
362 return;
363 }
364 unsigned WavesAfter = getMaxWaves(PressureAfter.first,
365 PressureAfter.second, MF);
366 unsigned WavesBefore = getMaxWaves(PressureBefore.first,
367 PressureBefore.second, MF);
368 DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
369 ", after " << WavesAfter << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
;
370
371 // We could not keep current target occupancy because of the just scheduled
372 // region. Record new occupancy for next scheduling cycle.
373 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
374 if (NewOccupancy < MinOccupancy) {
375 MinOccupancy = NewOccupancy;
376 DEBUG(dbgs() << "Occupancy lowered for the function to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
377 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
;
378 }
379
380 if (WavesAfter >= WavesBefore)
381 return;
382
383 DEBUG(dbgs() << "Attempting to revert scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Attempting to revert scheduling.\n"
; } } while (false)
;
384 RegionEnd = RegionBegin;
385 for (MachineInstr *MI : Unsched) {
386 if (MI->getIterator() != RegionEnd) {
387 BB->remove(MI);
388 BB->insert(RegionEnd, MI);
389 LIS->handleMove(*MI, true);
390 }
391 // Reset read-undef flags and update them later.
392 for (auto &Op : MI->operands())
393 if (Op.isReg() && Op.isDef())
394 Op.setIsUndef(false);
395 RegisterOperands RegOpers;
396 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
397 if (ShouldTrackLaneMasks) {
398 // Adjust liveness and add missing dead+read-undef flags.
399 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
400 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
401 } else {
402 // Adjust for missing dead-def flags.
403 RegOpers.detectDeadDefs(*MI, *LIS);
404 }
405 RegionEnd = MI->getIterator();
406 ++RegionEnd;
407 DEBUG(dbgs() << "Scheduling " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Scheduling " << *MI; } }
while (false)
;
408 }
409 RegionBegin = Unsched.front()->getIterator();
410
411 placeDebugValues();
412}
413
414static inline void setMask(const MachineRegisterInfo &MRI,
415 const SIRegisterInfo *SRI, unsigned Reg,
416 LaneBitmask &PrevMask, LaneBitmask NewMask,
417 unsigned &SGPRs, unsigned &VGPRs) {
418 int NewRegs = countPopulation(NewMask.getAsInteger()) -
419 countPopulation(PrevMask.getAsInteger());
420 if (SRI->isSGPRReg(MRI, Reg))
421 SGPRs += NewRegs;
422 if (SRI->isVGPR(MRI, Reg))
423 VGPRs += NewRegs;
424 assert ((int)SGPRs >= 0 && (int)VGPRs >= 0)(((int)SGPRs >= 0 && (int)VGPRs >= 0) ? static_cast
<void> (0) : __assert_fail ("(int)SGPRs >= 0 && (int)VGPRs >= 0"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 424, __PRETTY_FUNCTION__))
;
425 PrevMask = NewMask;
426}
427
428void GCNScheduleDAGMILive::discoverLiveIns() {
429 unsigned SGPRs = 0;
430 unsigned VGPRs = 0;
431
432 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
433 SlotIndex SI = LIS->getInstructionIndex(*begin()).getBaseIndex();
434 assert (SI.isValid())((SI.isValid()) ? static_cast<void> (0) : __assert_fail
("SI.isValid()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 434, __PRETTY_FUNCTION__))
;
435
436 DEBUG(dbgs() << "Region live-ins:")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Region live-ins:"; } } while (
false)
;
437 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
438 unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
439 if (MRI.reg_nodbg_empty(Reg))
440 continue;
441 const LiveInterval &LI = LIS->getInterval(Reg);
442 LaneBitmask LaneMask = LaneBitmask::getNone();
443 if (LI.hasSubRanges()) {
444 for (const auto &S : LI.subranges())
445 if (S.liveAt(SI))
446 LaneMask |= S.LaneMask;
447 } else if (LI.liveAt(SI)) {
448 LaneMask = MRI.getMaxLaneMaskForVReg(Reg);
449 }
450
451 if (LaneMask.any()) {
452 setMask(MRI, SRI, Reg, LiveIns[Reg], LaneMask, SGPRs, VGPRs);
453
454 DEBUG(dbgs() << ' ' << PrintVRegOrUnit(Reg, SRI) << ':'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << ' ' << PrintVRegOrUnit(Reg
, SRI) << ':' << PrintLaneMask(LiveIns[Reg]); } }
while (false)
455 << PrintLaneMask(LiveIns[Reg]))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << ' ' << PrintVRegOrUnit(Reg
, SRI) << ':' << PrintLaneMask(LiveIns[Reg]); } }
while (false)
;
456 }
457 }
458
459 LiveInPressure = std::make_pair(SGPRs, VGPRs);
460
461 DEBUG(dbgs() << "\nLive-in pressure:\nSGPR = " << SGPRsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "\nLive-in pressure:\nSGPR = "
<< SGPRs << "\nVGPR = " << VGPRs << '\n'
; } } while (false)
462 << "\nVGPR = " << VGPRs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "\nLive-in pressure:\nSGPR = "
<< SGPRs << "\nVGPR = " << VGPRs << '\n'
; } } while (false)
;
463}
464
465std::pair<unsigned, unsigned>
466GCNScheduleDAGMILive::getRealRegPressure() const {
467 unsigned SGPRs, MaxSGPRs, VGPRs, MaxVGPRs;
468 SGPRs = MaxSGPRs = LiveInPressure.first;
469 VGPRs = MaxVGPRs = LiveInPressure.second;
470
471 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
472 DenseMap<unsigned, LaneBitmask> LiveRegs(LiveIns);
473
474 for (const MachineInstr &MI : *this) {
475 if (MI.isDebugValue())
476 continue;
477 SlotIndex SI = LIS->getInstructionIndex(MI).getBaseIndex();
478 assert (SI.isValid())((SI.isValid()) ? static_cast<void> (0) : __assert_fail
("SI.isValid()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn298304/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 478, __PRETTY_FUNCTION__))
;
479
480 // Remove dead registers or mask bits.
481 for (auto &It : LiveRegs) {
482 if (It.second.none())
483 continue;
484 const LiveInterval &LI = LIS->getInterval(It.first);
485 if (LI.hasSubRanges()) {
486 for (const auto &S : LI.subranges())
487 if (!S.liveAt(SI))
488 setMask(MRI, SRI, It.first, It.second, It.second & ~S.LaneMask,
489 SGPRs, VGPRs);
490 } else if (!LI.liveAt(SI)) {
491 setMask(MRI, SRI, It.first, It.second, LaneBitmask::getNone(),
492 SGPRs, VGPRs);
493 }
494 }
495
496 // Add new registers or mask bits.
497 for (const auto &MO : MI.defs()) {
498 if (!MO.isReg())
499 continue;
500 unsigned Reg = MO.getReg();
501 if (!TargetRegisterInfo::isVirtualRegister(Reg))
502 continue;
503 unsigned SubRegIdx = MO.getSubReg();
504 LaneBitmask LaneMask = SubRegIdx != 0
505 ? TRI->getSubRegIndexLaneMask(SubRegIdx)
506 : MRI.getMaxLaneMaskForVReg(Reg);
507 LaneBitmask &LM = LiveRegs[Reg];
508 setMask(MRI, SRI, Reg, LM, LM | LaneMask, SGPRs, VGPRs);
509 }
510 MaxSGPRs = std::max(MaxSGPRs, SGPRs);
511 MaxVGPRs = std::max(MaxVGPRs, VGPRs);
512 }
513
514 DEBUG(dbgs() << "Real region's register pressure:\nSGPR = " << MaxSGPRsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Real region's register pressure:\nSGPR = "
<< MaxSGPRs << "\nVGPR = " << MaxVGPRs <<
'\n'; } } while (false)
515 << "\nVGPR = " << MaxVGPRs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Real region's register pressure:\nSGPR = "
<< MaxSGPRs << "\nVGPR = " << MaxVGPRs <<
'\n'; } } while (false)
;
516
517 return std::make_pair(MaxSGPRs, MaxVGPRs);
518}
519
520void GCNScheduleDAGMILive::finalizeSchedule() {
521 // Retry function scheduling if we found resulting occupancy and it is
522 // lower than used for first pass scheduling. This will give more freedom
523 // to schedule low register pressure blocks.
524 // Code is partially copied from MachineSchedulerBase::scheduleRegions().
525
526 if (!LIS || StartingOccupancy <= MinOccupancy)
1
Assuming the condition is false
2
Taking false branch
527 return;
528
529 DEBUG(dbgs() << "Retrying function scheduling with lowest recorded occupancy "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
530 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
;
531
532 Stage++;
533 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
534 S.TargetOccupancy = MinOccupancy;
535
536 MachineBasicBlock *MBB = nullptr;
3
'MBB' initialized to a null pointer value
537 for (auto Region : Regions) {
4
Assuming '__begin' is not equal to '__end'
538 RegionBegin = Region.first;
539 RegionEnd = Region.second;
540
541 if (RegionBegin->getParent() != MBB) {
5
Assuming the condition is false
6
Taking false branch
8
Assuming the condition is false
9
Taking false branch
11
Assuming the condition is false
12
Taking false branch
542 if (MBB) finishBlock();
543 MBB = RegionBegin->getParent();
544 startBlock(MBB);
545 }
546
547 unsigned NumRegionInstrs = std::distance(begin(), end());
548 enterRegion(MBB, begin(), end(), NumRegionInstrs);
549
550 // Skip empty scheduling regions (0 or 1 schedulable instructions).
551 if (begin() == end() || begin() == std::prev(end())) {
7
Taking false branch
10
Taking false branch
13
Taking false branch
552 exitRegion();
553 continue;
554 }
555 DEBUG(dbgs() << "********** MI Scheduling **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << "********** MI Scheduling **********\n"
; } } while (false)
;
556 DEBUG(dbgs() << MF.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
14
Within the expansion of the macro 'DEBUG':
a
Assuming 'DebugFlag' is not equal to 0
b
Assuming the condition is true
c
Called C++ object pointer is null
557 << ":BB#" << MBB->getNumber() << " " << MBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
558 << "\n From: " << *begin() << " To: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
559 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
560 else dbgs() << "End";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
561 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("misched")) { dbgs() << MF.getName() << ":BB#" <<
MBB->getNumber() << " " << MBB->getName() <<
"\n From: " << *begin() << " To: "; if (RegionEnd
!= MBB->end()) dbgs() << *RegionEnd; else dbgs() <<
"End"; dbgs() << " RegionInstrs: " << NumRegionInstrs
<< '\n'; } } while (false)
;
562
563 schedule();
564
565 exitRegion();
566 }
567 finishBlock();
568 LiveIns.shrink_and_clear();
569}