Bug Summary

File:lib/Target/AMDGPU/GCNSchedStrategy.cpp
Warning:line 532, column 7
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name GCNSchedStrategy.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Target/AMDGPU -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp -faddrsig
1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This contains a MachineSchedStrategy implementation for maximizing wave
12/// occupancy on GCN hardware.
13//===----------------------------------------------------------------------===//
14
15#include "GCNSchedStrategy.h"
16#include "AMDGPUSubtarget.h"
17#include "SIInstrInfo.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIRegisterInfo.h"
20#include "llvm/CodeGen/RegisterClassInfo.h"
21#include "llvm/Support/MathExtras.h"
22
23#define DEBUG_TYPE"machine-scheduler" "machine-scheduler"
24
25using namespace llvm;
26
27GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
28 const MachineSchedContext *C) :
29 GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
30
31void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
32 GenericScheduler::initialize(DAG);
33
34 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
35
36 MF = &DAG->MF;
37
38 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
39
40 // FIXME: This is also necessary, because some passes that run after
41 // scheduling and before regalloc increase register pressure.
42 const int ErrorMargin = 3;
43
44 SGPRExcessLimit = Context->RegClassInfo
45 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
46 VGPRExcessLimit = Context->RegClassInfo
47 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
48 if (TargetOccupancy) {
49 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
50 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
51 } else {
52 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
53 SRI->getSGPRPressureSet());
54 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
55 SRI->getVGPRPressureSet());
56 }
57
58 SGPRCriticalLimit -= ErrorMargin;
59 VGPRCriticalLimit -= ErrorMargin;
60}
61
62void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
63 bool AtTop, const RegPressureTracker &RPTracker,
64 const SIRegisterInfo *SRI,
65 unsigned SGPRPressure,
66 unsigned VGPRPressure) {
67
68 Cand.SU = SU;
69 Cand.AtTop = AtTop;
70
71 // getDownwardPressure() and getUpwardPressure() make temporary changes to
72 // the tracker, so we need to pass those function a non-const copy.
73 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
74
75 std::vector<unsigned> Pressure;
76 std::vector<unsigned> MaxPressure;
77
78 if (AtTop)
79 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
80 else {
81 // FIXME: I think for bottom up scheduling, the register pressure is cached
82 // and can be retrieved by DAG->getPressureDif(SU).
83 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
84 }
85
86 unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
87 unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
88
89 // If two instructions increase the pressure of different register sets
90 // by the same amount, the generic scheduler will prefer to schedule the
91 // instruction that increases the set with the least amount of registers,
92 // which in our case would be SGPRs. This is rarely what we want, so
93 // when we report excess/critical register pressure, we do it either
94 // only for VGPRs or only for SGPRs.
95
96 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
97 const unsigned MaxVGPRPressureInc = 16;
98 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
99 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
100
101
102 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
103 // to increase the likelihood we don't go over the limits. We should improve
104 // the analysis to look through dependencies to find the path with the least
105 // register pressure.
106
107 // We only need to update the RPDelata for instructions that increase
108 // register pressure. Instructions that decrease or keep reg pressure
109 // the same will be marked as RegExcess in tryCandidate() when they
110 // are compared with instructions that increase the register pressure.
111 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
112 Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
113 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
114 }
115
116 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
117 Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
118 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
119 }
120
121 // Register pressure is considered 'CRITICAL' if it is approaching a value
122 // that would reduce the wave occupancy for the execution unit. When
123 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
124 // has the same cost, so we don't need to prefer one over the other.
125
126 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
127 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
128
129 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
130 if (SGPRDelta > VGPRDelta) {
131 Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
132 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
133 } else {
134 Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
135 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
136 }
137 }
138}
139
140// This function is mostly cut and pasted from
141// GenericScheduler::pickNodeFromQueue()
142void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
143 const CandPolicy &ZonePolicy,
144 const RegPressureTracker &RPTracker,
145 SchedCandidate &Cand) {
146 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
147 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
148 unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
149 unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
150 ReadyQueue &Q = Zone.Available;
151 for (SUnit *SU : Q) {
152
153 SchedCandidate TryCand(ZonePolicy);
154 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
155 SGPRPressure, VGPRPressure);
156 // Pass SchedBoundary only when comparing nodes from the same boundary.
157 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
158 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
159 if (TryCand.Reason != NoCand) {
160 // Initialize resource delta if needed in case future heuristics query it.
161 if (TryCand.ResDelta == SchedResourceDelta())
162 TryCand.initResourceDelta(Zone.DAG, SchedModel);
163 Cand.setBest(TryCand);
164 }
165 }
166}
167
168// This function is mostly cut and pasted from
169// GenericScheduler::pickNodeBidirectional()
170SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
171 // Schedule as far as possible in the direction of no choice. This is most
172 // efficient, but also provides the best heuristics for CriticalPSets.
173 if (SUnit *SU = Bot.pickOnlyChoice()) {
174 IsTopNode = false;
175 return SU;
176 }
177 if (SUnit *SU = Top.pickOnlyChoice()) {
178 IsTopNode = true;
179 return SU;
180 }
181 // Set the bottom-up policy based on the state of the current bottom zone and
182 // the instructions outside the zone, including the top zone.
183 CandPolicy BotPolicy;
184 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
185 // Set the top-down policy based on the state of the current top zone and
186 // the instructions outside the zone, including the bottom zone.
187 CandPolicy TopPolicy;
188 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
189
190 // See if BotCand is still valid (because we previously scheduled from Top).
191 LLVM_DEBUG(dbgs() << "Picking from Bot:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Bot:\n"
; } } while (false)
;
192 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
193 BotCand.Policy != BotPolicy) {
194 BotCand.reset(CandPolicy());
195 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
196 assert(BotCand.Reason != NoCand && "failed to find the first candidate")((BotCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 196, __PRETTY_FUNCTION__))
;
197 } else {
198 LLVM_DEBUG(traceCandidate(BotCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(BotCand); } } while (
false)
;
199 }
200
201 // Check if the top Q has a better candidate.
202 LLVM_DEBUG(dbgs() << "Picking from Top:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Top:\n"
; } } while (false)
;
203 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
204 TopCand.Policy != TopPolicy) {
205 TopCand.reset(CandPolicy());
206 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
207 assert(TopCand.Reason != NoCand && "failed to find the first candidate")((TopCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 207, __PRETTY_FUNCTION__))
;
208 } else {
209 LLVM_DEBUG(traceCandidate(TopCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(TopCand); } } while (
false)
;
210 }
211
212 // Pick best from BotCand and TopCand.
213 LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false)
214 dbgs() << "Bot Cand: "; traceCandidate(BotCand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false)
;
215 SchedCandidate Cand;
216 if (TopCand.Reason == BotCand.Reason) {
217 Cand = BotCand;
218 GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
219 TopCand.Reason = NoCand;
220 GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
221 if (TopCand.Reason != NoCand) {
222 Cand.setBest(TopCand);
223 } else {
224 TopCand.Reason = TopReason;
225 }
226 } else {
227 if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
228 Cand = TopCand;
229 } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
230 Cand = BotCand;
231 } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
232 Cand = TopCand;
233 } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
234 Cand = BotCand;
235 } else {
236 if (BotCand.Reason > TopCand.Reason) {
237 Cand = TopCand;
238 } else {
239 Cand = BotCand;
240 }
241 }
242 }
243 LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking: "; traceCandidate
(Cand);; } } while (false)
;
244
245 IsTopNode = Cand.AtTop;
246 return Cand.SU;
247}
248
249// This function is mostly cut and pasted from
250// GenericScheduler::pickNode()
251SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
252 if (DAG->top() == DAG->bottom()) {
253 assert(Top.Available.empty() && Top.Pending.empty() &&((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 254, __PRETTY_FUNCTION__))
254 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage")((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 254, __PRETTY_FUNCTION__))
;
255 return nullptr;
256 }
257 SUnit *SU;
258 do {
259 if (RegionPolicy.OnlyTopDown) {
260 SU = Top.pickOnlyChoice();
261 if (!SU) {
262 CandPolicy NoPolicy;
263 TopCand.reset(NoPolicy);
264 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
265 assert(TopCand.Reason != NoCand && "failed to find a candidate")((TopCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 265, __PRETTY_FUNCTION__))
;
266 SU = TopCand.SU;
267 }
268 IsTopNode = true;
269 } else if (RegionPolicy.OnlyBottomUp) {
270 SU = Bot.pickOnlyChoice();
271 if (!SU) {
272 CandPolicy NoPolicy;
273 BotCand.reset(NoPolicy);
274 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
275 assert(BotCand.Reason != NoCand && "failed to find a candidate")((BotCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 275, __PRETTY_FUNCTION__))
;
276 SU = BotCand.SU;
277 }
278 IsTopNode = false;
279 } else {
280 SU = pickNodeBidirectional(IsTopNode);
281 }
282 } while (SU->isScheduled);
283
284 if (SU->isTopReady())
285 Top.removeReady(SU);
286 if (SU->isBottomReady())
287 Bot.removeReady(SU);
288
289 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false)
290 << *SU->getInstr())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false)
;
291 return SU;
292}
293
294GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
295 std::unique_ptr<MachineSchedStrategy> S) :
296 ScheduleDAGMILive(C, std::move(S)),
297 ST(MF.getSubtarget<GCNSubtarget>()),
298 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
299 StartingOccupancy(MFI.getOccupancy()),
300 MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
301
302 LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Starting occupancy is "
<< StartingOccupancy << ".\n"; } } while (false)
;
303}
304
305void GCNScheduleDAGMILive::schedule() {
306 if (Stage == 0) {
307 // Just record regions at the first pass.
308 Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
309 return;
310 }
311
312 std::vector<MachineInstr*> Unsched;
313 Unsched.reserve(NumRegionInstrs);
314 for (auto &I : *this) {
315 Unsched.push_back(&I);
316 }
317
318 GCNRegPressure PressureBefore;
319 if (LIS) {
320 PressureBefore = Pressure[RegionIdx];
321
322 LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
323 GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
324 dbgs() << "Region live-in pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
325 llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
326 dbgs() << "Region register pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
327 PressureBefore.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
;
328 }
329
330 ScheduleDAGMILive::schedule();
331 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
332
333 if (!LIS)
334 return;
335
336 // Check the results of scheduling.
337 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
338 auto PressureAfter = getRealRegPressure();
339
340 LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false)
341 PressureAfter.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false)
;
342
343 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
344 PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
345 Pressure[RegionIdx] = PressureAfter;
346 LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure in desired limits, done.\n"
; } } while (false)
;
347 return;
348 }
349 unsigned Occ = MFI.getOccupancy();
350 unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
351 unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
352 LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBeforedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
353 << ", after " << WavesAfter << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
;
354
355 // We could not keep current target occupancy because of the just scheduled
356 // region. Record new occupancy for next scheduling cycle.
357 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
358 // Allow memory bound functions to drop to 4 waves if not limited by an
359 // attribute.
360 if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
361 WavesAfter >= MFI.getMinAllowedOccupancy()) {
362 LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false)
363 << MFI.getMinAllowedOccupancy() << " waves\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false)
;
364 NewOccupancy = WavesAfter;
365 }
366 if (NewOccupancy < MinOccupancy) {
367 MinOccupancy = NewOccupancy;
368 MFI.limitOccupancy(MinOccupancy);
369 LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
370 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
;
371 }
372
373 if (WavesAfter >= MinOccupancy) {
374 Pressure[RegionIdx] = PressureAfter;
375 return;
376 }
377
378 LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Attempting to revert scheduling.\n"
; } } while (false)
;
379 RegionEnd = RegionBegin;
380 for (MachineInstr *MI : Unsched) {
381 if (MI->isDebugInstr())
382 continue;
383
384 if (MI->getIterator() != RegionEnd) {
385 BB->remove(MI);
386 BB->insert(RegionEnd, MI);
387 if (!MI->isDebugInstr())
388 LIS->handleMove(*MI, true);
389 }
390 // Reset read-undef flags and update them later.
391 for (auto &Op : MI->operands())
392 if (Op.isReg() && Op.isDef())
393 Op.setIsUndef(false);
394 RegisterOperands RegOpers;
395 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
396 if (!MI->isDebugInstr()) {
397 if (ShouldTrackLaneMasks) {
398 // Adjust liveness and add missing dead+read-undef flags.
399 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
400 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
401 } else {
402 // Adjust for missing dead-def flags.
403 RegOpers.detectDeadDefs(*MI, *LIS);
404 }
405 }
406 RegionEnd = MI->getIterator();
407 ++RegionEnd;
408 LLVM_DEBUG(dbgs() << "Scheduling " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling " <<
*MI; } } while (false)
;
409 }
410 RegionBegin = Unsched.front()->getIterator();
411 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
412
413 placeDebugValues();
414}
415
416GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
417 GCNDownwardRPTracker RPTracker(*LIS);
418 RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
419 return RPTracker.moveMaxPressure();
420}
421
422void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
423 GCNDownwardRPTracker RPTracker(*LIS);
424
425 // If the block has the only successor then live-ins of that successor are
426 // live-outs of the current block. We can reuse calculated live set if the
427 // successor will be sent to scheduling past current block.
428 const MachineBasicBlock *OnlySucc = nullptr;
429 if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
430 SlotIndexes *Ind = LIS->getSlotIndexes();
431 if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
432 OnlySucc = *MBB->succ_begin();
433 }
434
435 // Scheduler sends regions from the end of the block upwards.
436 size_t CurRegion = RegionIdx;
437 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
438 if (Regions[CurRegion].first->getParent() != MBB)
439 break;
440 --CurRegion;
441
442 auto I = MBB->begin();
443 auto LiveInIt = MBBLiveIns.find(MBB);
444 if (LiveInIt != MBBLiveIns.end()) {
445 auto LiveIn = std::move(LiveInIt->second);
446 RPTracker.reset(*MBB->begin(), &LiveIn);
447 MBBLiveIns.erase(LiveInIt);
448 } else {
449 I = Regions[CurRegion].first;
450 RPTracker.reset(*I);
451 }
452
453 for ( ; ; ) {
454 I = RPTracker.getNext();
455
456 if (Regions[CurRegion].first == I) {
457 LiveIns[CurRegion] = RPTracker.getLiveRegs();
458 RPTracker.clearMaxPressure();
459 }
460
461 if (Regions[CurRegion].second == I) {
462 Pressure[CurRegion] = RPTracker.moveMaxPressure();
463 if (CurRegion-- == RegionIdx)
464 break;
465 }
466 RPTracker.advanceToNext();
467 RPTracker.advanceBeforeNext();
468 }
469
470 if (OnlySucc) {
471 if (I != MBB->end()) {
472 RPTracker.advanceToNext();
473 RPTracker.advance(MBB->end());
474 }
475 RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
476 RPTracker.advanceBeforeNext();
477 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
478 }
479}
480
481void GCNScheduleDAGMILive::finalizeSchedule() {
482 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
483 LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "All regions recorded, starting actual scheduling.\n"
; } } while (false)
;
484
485 LiveIns.resize(Regions.size());
486 Pressure.resize(Regions.size());
487
488 do {
489 Stage++;
490 RegionIdx = 0;
491 MachineBasicBlock *MBB = nullptr;
1
'MBB' initialized to a null pointer value
492
493 if (Stage > 1) {
2
Assuming the condition is false
3
Taking false branch
494 // Retry function scheduling if we found resulting occupancy and it is
495 // lower than used for first pass scheduling. This will give more freedom
496 // to schedule low register pressure blocks.
497 // Code is partially copied from MachineSchedulerBase::scheduleRegions().
498
499 if (!LIS || StartingOccupancy <= MinOccupancy)
500 break;
501
502 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
503 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
504 << "Retrying function scheduling with lowest recorded occupancy "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
505 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
;
506
507 S.setTargetOccupancy(MinOccupancy);
508 }
509
510 for (auto Region : Regions) {
4
Assuming '__begin2' is not equal to '__end2'
511 RegionBegin = Region.first;
512 RegionEnd = Region.second;
513
514 if (RegionBegin->getParent() != MBB) {
5
Assuming the condition is false
6
Taking false branch
515 if (MBB) finishBlock();
516 MBB = RegionBegin->getParent();
517 startBlock(MBB);
518 if (Stage == 1)
519 computeBlockPressure(MBB);
520 }
521
522 unsigned NumRegionInstrs = std::distance(begin(), end());
523 enterRegion(MBB, begin(), end(), NumRegionInstrs);
524
525 // Skip empty scheduling regions (0 or 1 schedulable instructions).
526 if (begin() == end() || begin() == std::prev(end())) {
7
Taking false branch
527 exitRegion();
528 continue;
529 }
530
531 LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "********** MI Scheduling **********\n"
; } } while (false)
;
532 LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
8
Within the expansion of the macro 'LLVM_DEBUG':
a
Assuming 'DebugFlag' is not equal to 0
b
Assuming the condition is true
c
Forming reference to null pointer
533 << MBB->getName() << "\n From: " << *begin()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
534 << " To: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
535 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
536 else dbgs() << "End";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
537 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
;
538
539 schedule();
540
541 exitRegion();
542 ++RegionIdx;
543 }
544 finishBlock();
545
546 } while (Stage < 2);
547}