Bug Summary

File:llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
Warning:line 627, column 7
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name GCNSchedStrategy.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/AMDGPU -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/AMDGPU -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-07-26-235520-9401-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This contains a MachineSchedStrategy implementation for maximizing wave
11/// occupancy on GCN hardware.
12//===----------------------------------------------------------------------===//
13
14#include "GCNSchedStrategy.h"
15#include "SIMachineFunctionInfo.h"
16
17#define DEBUG_TYPE"machine-scheduler" "machine-scheduler"
18
19using namespace llvm;
20
21GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
22 const MachineSchedContext *C) :
23 GenericScheduler(C), TargetOccupancy(0), HasClusteredNodes(false),
24 HasExcessPressure(false), MF(nullptr) { }
25
26void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
27 GenericScheduler::initialize(DAG);
28
29 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
30
31 MF = &DAG->MF;
32
33 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
34
35 // FIXME: This is also necessary, because some passes that run after
36 // scheduling and before regalloc increase register pressure.
37 const int ErrorMargin = 3;
38
39 SGPRExcessLimit = Context->RegClassInfo
40 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
41 VGPRExcessLimit = Context->RegClassInfo
42 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
43 if (TargetOccupancy) {
44 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
45 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
46 } else {
47 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
48 AMDGPU::RegisterPressureSets::SReg_32);
49 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
50 AMDGPU::RegisterPressureSets::VGPR_32);
51 }
52
53 SGPRCriticalLimit -= ErrorMargin;
54 VGPRCriticalLimit -= ErrorMargin;
55}
56
57void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
58 bool AtTop, const RegPressureTracker &RPTracker,
59 const SIRegisterInfo *SRI,
60 unsigned SGPRPressure,
61 unsigned VGPRPressure) {
62
63 Cand.SU = SU;
64 Cand.AtTop = AtTop;
65
66 // getDownwardPressure() and getUpwardPressure() make temporary changes to
67 // the tracker, so we need to pass those function a non-const copy.
68 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
69
70 Pressure.clear();
71 MaxPressure.clear();
72
73 if (AtTop)
74 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
75 else {
76 // FIXME: I think for bottom up scheduling, the register pressure is cached
77 // and can be retrieved by DAG->getPressureDif(SU).
78 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
79 }
80
81 unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
82 unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
83
84 // If two instructions increase the pressure of different register sets
85 // by the same amount, the generic scheduler will prefer to schedule the
86 // instruction that increases the set with the least amount of registers,
87 // which in our case would be SGPRs. This is rarely what we want, so
88 // when we report excess/critical register pressure, we do it either
89 // only for VGPRs or only for SGPRs.
90
91 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
92 const unsigned MaxVGPRPressureInc = 16;
93 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
94 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
95
96
97 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
98 // to increase the likelihood we don't go over the limits. We should improve
99 // the analysis to look through dependencies to find the path with the least
100 // register pressure.
101
102 // We only need to update the RPDelta for instructions that increase register
103 // pressure. Instructions that decrease or keep reg pressure the same will be
104 // marked as RegExcess in tryCandidate() when they are compared with
105 // instructions that increase the register pressure.
106 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
107 HasExcessPressure = true;
108 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
109 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
110 }
111
112 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
113 HasExcessPressure = true;
114 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
115 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
116 }
117
118 // Register pressure is considered 'CRITICAL' if it is approaching a value
119 // that would reduce the wave occupancy for the execution unit. When
120 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
121 // has the same cost, so we don't need to prefer one over the other.
122
123 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
124 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
125
126 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
127 HasExcessPressure = true;
128 if (SGPRDelta > VGPRDelta) {
129 Cand.RPDelta.CriticalMax =
130 PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
131 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
132 } else {
133 Cand.RPDelta.CriticalMax =
134 PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
135 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
136 }
137 }
138}
139
140// This function is mostly cut and pasted from
141// GenericScheduler::pickNodeFromQueue()
142void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
143 const CandPolicy &ZonePolicy,
144 const RegPressureTracker &RPTracker,
145 SchedCandidate &Cand) {
146 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
147 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
148 unsigned SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
149 unsigned VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
150 ReadyQueue &Q = Zone.Available;
151 for (SUnit *SU : Q) {
152
153 SchedCandidate TryCand(ZonePolicy);
154 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
155 SGPRPressure, VGPRPressure);
156 // Pass SchedBoundary only when comparing nodes from the same boundary.
157 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
158 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
159 if (TryCand.Reason != NoCand) {
160 // Initialize resource delta if needed in case future heuristics query it.
161 if (TryCand.ResDelta == SchedResourceDelta())
162 TryCand.initResourceDelta(Zone.DAG, SchedModel);
163 Cand.setBest(TryCand);
164 LLVM_DEBUG(traceCandidate(Cand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(Cand); } } while (false
)
;
165 }
166 }
167}
168
169// This function is mostly cut and pasted from
170// GenericScheduler::pickNodeBidirectional()
171SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
172 // Schedule as far as possible in the direction of no choice. This is most
173 // efficient, but also provides the best heuristics for CriticalPSets.
174 if (SUnit *SU = Bot.pickOnlyChoice()) {
175 IsTopNode = false;
176 return SU;
177 }
178 if (SUnit *SU = Top.pickOnlyChoice()) {
179 IsTopNode = true;
180 return SU;
181 }
182 // Set the bottom-up policy based on the state of the current bottom zone and
183 // the instructions outside the zone, including the top zone.
184 CandPolicy BotPolicy;
185 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
186 // Set the top-down policy based on the state of the current top zone and
187 // the instructions outside the zone, including the bottom zone.
188 CandPolicy TopPolicy;
189 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
190
191 // See if BotCand is still valid (because we previously scheduled from Top).
192 LLVM_DEBUG(dbgs() << "Picking from Bot:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Bot:\n"
; } } while (false)
;
193 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
194 BotCand.Policy != BotPolicy) {
195 BotCand.reset(CandPolicy());
196 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
197 assert(BotCand.Reason != NoCand && "failed to find the first candidate")(static_cast <bool> (BotCand.Reason != NoCand &&
"failed to find the first candidate") ? void (0) : __assert_fail
("BotCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 197, __extension__ __PRETTY_FUNCTION__))
;
198 } else {
199 LLVM_DEBUG(traceCandidate(BotCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(BotCand); } } while (
false)
;
200#ifndef NDEBUG
201 if (VerifyScheduling) {
202 SchedCandidate TCand;
203 TCand.reset(CandPolicy());
204 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
205 assert(TCand.SU == BotCand.SU &&(static_cast <bool> (TCand.SU == BotCand.SU && "Last pick result should correspond to re-picking right now"
) ? void (0) : __assert_fail ("TCand.SU == BotCand.SU && \"Last pick result should correspond to re-picking right now\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 206, __extension__ __PRETTY_FUNCTION__))
206 "Last pick result should correspond to re-picking right now")(static_cast <bool> (TCand.SU == BotCand.SU && "Last pick result should correspond to re-picking right now"
) ? void (0) : __assert_fail ("TCand.SU == BotCand.SU && \"Last pick result should correspond to re-picking right now\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 206, __extension__ __PRETTY_FUNCTION__))
;
207 }
208#endif
209 }
210
211 // Check if the top Q has a better candidate.
212 LLVM_DEBUG(dbgs() << "Picking from Top:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Top:\n"
; } } while (false)
;
213 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
214 TopCand.Policy != TopPolicy) {
215 TopCand.reset(CandPolicy());
216 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
217 assert(TopCand.Reason != NoCand && "failed to find the first candidate")(static_cast <bool> (TopCand.Reason != NoCand &&
"failed to find the first candidate") ? void (0) : __assert_fail
("TopCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 217, __extension__ __PRETTY_FUNCTION__))
;
218 } else {
219 LLVM_DEBUG(traceCandidate(TopCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(TopCand); } } while (
false)
;
220#ifndef NDEBUG
221 if (VerifyScheduling) {
222 SchedCandidate TCand;
223 TCand.reset(CandPolicy());
224 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
225 assert(TCand.SU == TopCand.SU &&(static_cast <bool> (TCand.SU == TopCand.SU && "Last pick result should correspond to re-picking right now"
) ? void (0) : __assert_fail ("TCand.SU == TopCand.SU && \"Last pick result should correspond to re-picking right now\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 226, __extension__ __PRETTY_FUNCTION__))
226 "Last pick result should correspond to re-picking right now")(static_cast <bool> (TCand.SU == TopCand.SU && "Last pick result should correspond to re-picking right now"
) ? void (0) : __assert_fail ("TCand.SU == TopCand.SU && \"Last pick result should correspond to re-picking right now\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 226, __extension__ __PRETTY_FUNCTION__))
;
227 }
228#endif
229 }
230
231 // Pick best from BotCand and TopCand.
232 LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false)
233 dbgs() << "Bot Cand: "; traceCandidate(BotCand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false)
;
234 SchedCandidate Cand = BotCand;
235 TopCand.Reason = NoCand;
236 GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
237 if (TopCand.Reason != NoCand) {
238 Cand.setBest(TopCand);
239 }
240 LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking: "; traceCandidate
(Cand);; } } while (false)
;
241
242 IsTopNode = Cand.AtTop;
243 return Cand.SU;
244}
245
246// This function is mostly cut and pasted from
247// GenericScheduler::pickNode()
248SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
249 if (DAG->top() == DAG->bottom()) {
250 assert(Top.Available.empty() && Top.Pending.empty() &&(static_cast <bool> (Top.Available.empty() && Top
.Pending.empty() && Bot.Available.empty() && Bot
.Pending.empty() && "ReadyQ garbage") ? void (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 251, __extension__ __PRETTY_FUNCTION__))
251 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage")(static_cast <bool> (Top.Available.empty() && Top
.Pending.empty() && Bot.Available.empty() && Bot
.Pending.empty() && "ReadyQ garbage") ? void (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 251, __extension__ __PRETTY_FUNCTION__))
;
252 return nullptr;
253 }
254 SUnit *SU;
255 do {
256 if (RegionPolicy.OnlyTopDown) {
257 SU = Top.pickOnlyChoice();
258 if (!SU) {
259 CandPolicy NoPolicy;
260 TopCand.reset(NoPolicy);
261 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
262 assert(TopCand.Reason != NoCand && "failed to find a candidate")(static_cast <bool> (TopCand.Reason != NoCand &&
"failed to find a candidate") ? void (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 262, __extension__ __PRETTY_FUNCTION__))
;
263 SU = TopCand.SU;
264 }
265 IsTopNode = true;
266 } else if (RegionPolicy.OnlyBottomUp) {
267 SU = Bot.pickOnlyChoice();
268 if (!SU) {
269 CandPolicy NoPolicy;
270 BotCand.reset(NoPolicy);
271 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
272 assert(BotCand.Reason != NoCand && "failed to find a candidate")(static_cast <bool> (BotCand.Reason != NoCand &&
"failed to find a candidate") ? void (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 272, __extension__ __PRETTY_FUNCTION__))
;
273 SU = BotCand.SU;
274 }
275 IsTopNode = false;
276 } else {
277 SU = pickNodeBidirectional(IsTopNode);
278 }
279 } while (SU->isScheduled);
280
281 if (SU->isTopReady())
282 Top.removeReady(SU);
283 if (SU->isBottomReady())
284 Bot.removeReady(SU);
285
286 if (!HasClusteredNodes && SU->getInstr()->mayLoadOrStore()) {
287 for (SDep &Dep : SU->Preds) {
288 if (Dep.isCluster()) {
289 HasClusteredNodes = true;
290 break;
291 }
292 }
293 }
294
295 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false)
296 << *SU->getInstr())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false)
;
297 return SU;
298}
299
300GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
301 std::unique_ptr<MachineSchedStrategy> S) :
302 ScheduleDAGMILive(C, std::move(S)),
303 ST(MF.getSubtarget<GCNSubtarget>()),
304 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
305 StartingOccupancy(MFI.getOccupancy()),
306 MinOccupancy(StartingOccupancy), Stage(Collect), RegionIdx(0) {
307
308 LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Starting occupancy is "
<< StartingOccupancy << ".\n"; } } while (false)
;
309}
310
311void GCNScheduleDAGMILive::schedule() {
312 if (Stage == Collect) {
313 // Just record regions at the first pass.
314 Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
315 return;
316 }
317
318 std::vector<MachineInstr*> Unsched;
319 Unsched.reserve(NumRegionInstrs);
320 for (auto &I : *this) {
321 Unsched.push_back(&I);
322 }
323
324 GCNRegPressure PressureBefore;
325 if (LIS) {
326 PressureBefore = Pressure[RegionIdx];
327
328 LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
329 GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
330 dbgs() << "Region live-in pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
331 llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
332 dbgs() << "Region register pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
333 PressureBefore.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
;
334 }
335
336 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
337 // Set HasClusteredNodes to true for late stages where we have already
338 // collected it. That way pickNode() will not scan SDep's when not needed.
339 S.HasClusteredNodes = Stage > InitialSchedule;
340 S.HasExcessPressure = false;
341 ScheduleDAGMILive::schedule();
342 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
343 RescheduleRegions[RegionIdx] = false;
344 if (Stage == InitialSchedule && S.HasClusteredNodes)
345 RegionsWithClusters[RegionIdx] = true;
346 if (S.HasExcessPressure)
347 RegionsWithHighRP[RegionIdx] = true;
348
349 if (!LIS)
350 return;
351
352 // Check the results of scheduling.
353 auto PressureAfter = getRealRegPressure();
354
355 LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false)
356 PressureAfter.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false)
;
357
358 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
359 PressureAfter.getVGPRNum(ST.hasGFX90AInsts()) <= S.VGPRCriticalLimit) {
360 Pressure[RegionIdx] = PressureAfter;
361 LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure in desired limits, done.\n"
; } } while (false)
;
362 return;
363 }
364 unsigned Occ = MFI.getOccupancy();
365 unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
366 unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
367 LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBeforedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
368 << ", after " << WavesAfter << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
;
369
370 // We could not keep current target occupancy because of the just scheduled
371 // region. Record new occupancy for next scheduling cycle.
372 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
373 // Allow memory bound functions to drop to 4 waves if not limited by an
374 // attribute.
375 if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
376 WavesAfter >= MFI.getMinAllowedOccupancy()) {
377 LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false)
378 << MFI.getMinAllowedOccupancy() << " waves\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false)
;
379 NewOccupancy = WavesAfter;
380 }
381 if (NewOccupancy < MinOccupancy) {
382 MinOccupancy = NewOccupancy;
383 MFI.limitOccupancy(MinOccupancy);
384 LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
385 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
;
386 }
387
388 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
389 unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
390 if (PressureAfter.getVGPRNum(false) > MaxVGPRs ||
391 PressureAfter.getAGPRNum() > MaxVGPRs ||
392 PressureAfter.getSGPRNum() > MaxSGPRs) {
393 RescheduleRegions[RegionIdx] = true;
394 RegionsWithHighRP[RegionIdx] = true;
395 }
396
397 if (WavesAfter >= MinOccupancy) {
398 if (Stage == UnclusteredReschedule &&
399 !PressureAfter.less(ST, PressureBefore)) {
400 LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Unclustered reschedule did not help.\n"
; } } while (false)
;
401 } else if (WavesAfter > MFI.getMinWavesPerEU() ||
402 PressureAfter.less(ST, PressureBefore) ||
403 !RescheduleRegions[RegionIdx]) {
404 Pressure[RegionIdx] = PressureAfter;
405 if (!RegionsWithClusters[RegionIdx] &&
406 (Stage + 1) == UnclusteredReschedule)
407 RescheduleRegions[RegionIdx] = false;
408 return;
409 } else {
410 LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "New pressure will result in more spilling.\n"
; } } while (false)
;
411 }
412 }
413
414 LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Attempting to revert scheduling.\n"
; } } while (false)
;
415 RescheduleRegions[RegionIdx] = RegionsWithClusters[RegionIdx] ||
416 (Stage + 1) != UnclusteredReschedule;
417 RegionEnd = RegionBegin;
418 for (MachineInstr *MI : Unsched) {
419 if (MI->isDebugInstr())
420 continue;
421
422 if (MI->getIterator() != RegionEnd) {
423 BB->remove(MI);
424 BB->insert(RegionEnd, MI);
425 if (!MI->isDebugInstr())
426 LIS->handleMove(*MI, true);
427 }
428 // Reset read-undef flags and update them later.
429 for (auto &Op : MI->operands())
430 if (Op.isReg() && Op.isDef())
431 Op.setIsUndef(false);
432 RegisterOperands RegOpers;
433 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
434 if (!MI->isDebugInstr()) {
435 if (ShouldTrackLaneMasks) {
436 // Adjust liveness and add missing dead+read-undef flags.
437 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
438 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
439 } else {
440 // Adjust for missing dead-def flags.
441 RegOpers.detectDeadDefs(*MI, *LIS);
442 }
443 }
444 RegionEnd = MI->getIterator();
445 ++RegionEnd;
446 LLVM_DEBUG(dbgs() << "Scheduling " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling " <<
*MI; } } while (false)
;
447 }
448 RegionBegin = Unsched.front()->getIterator();
449 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
450
451 placeDebugValues();
452}
453
454GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
455 GCNDownwardRPTracker RPTracker(*LIS);
456 RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
457 return RPTracker.moveMaxPressure();
458}
459
460void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
461 GCNDownwardRPTracker RPTracker(*LIS);
462
463 // If the block has the only successor then live-ins of that successor are
464 // live-outs of the current block. We can reuse calculated live set if the
465 // successor will be sent to scheduling past current block.
466 const MachineBasicBlock *OnlySucc = nullptr;
467 if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
468 SlotIndexes *Ind = LIS->getSlotIndexes();
469 if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
470 OnlySucc = *MBB->succ_begin();
471 }
472
473 // Scheduler sends regions from the end of the block upwards.
474 size_t CurRegion = RegionIdx;
475 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
476 if (Regions[CurRegion].first->getParent() != MBB)
477 break;
478 --CurRegion;
479
480 auto I = MBB->begin();
481 auto LiveInIt = MBBLiveIns.find(MBB);
482 if (LiveInIt != MBBLiveIns.end()) {
483 auto LiveIn = std::move(LiveInIt->second);
484 RPTracker.reset(*MBB->begin(), &LiveIn);
485 MBBLiveIns.erase(LiveInIt);
486 } else {
487 auto &Rgn = Regions[CurRegion];
488 I = Rgn.first;
489 auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second);
490 auto LRS = BBLiveInMap.lookup(NonDbgMI);
491#ifdef EXPENSIVE_CHECKS
492 assert(isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS))(static_cast <bool> (isEqual(getLiveRegsBefore(*NonDbgMI
, *LIS), LRS)) ? void (0) : __assert_fail ("isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS)"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 492, __extension__ __PRETTY_FUNCTION__))
;
493#endif
494 RPTracker.reset(*I, &LRS);
495 }
496
497 for ( ; ; ) {
498 I = RPTracker.getNext();
499
500 if (Regions[CurRegion].first == I) {
501 LiveIns[CurRegion] = RPTracker.getLiveRegs();
502 RPTracker.clearMaxPressure();
503 }
504
505 if (Regions[CurRegion].second == I) {
506 Pressure[CurRegion] = RPTracker.moveMaxPressure();
507 if (CurRegion-- == RegionIdx)
508 break;
509 }
510 RPTracker.advanceToNext();
511 RPTracker.advanceBeforeNext();
512 }
513
514 if (OnlySucc) {
515 if (I != MBB->end()) {
516 RPTracker.advanceToNext();
517 RPTracker.advance(MBB->end());
518 }
519 RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
520 RPTracker.advanceBeforeNext();
521 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
522 }
523}
524
525DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
526GCNScheduleDAGMILive::getBBLiveInMap() const {
527 assert(!Regions.empty())(static_cast <bool> (!Regions.empty()) ? void (0) : __assert_fail
("!Regions.empty()", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 527, __extension__ __PRETTY_FUNCTION__))
;
528 std::vector<MachineInstr *> BBStarters;
529 BBStarters.reserve(Regions.size());
530 auto I = Regions.rbegin(), E = Regions.rend();
531 auto *BB = I->first->getParent();
532 do {
533 auto *MI = &*skipDebugInstructionsForward(I->first, I->second);
534 BBStarters.push_back(MI);
535 do {
536 ++I;
537 } while (I != E && I->first->getParent() == BB);
538 } while (I != E);
539 return getLiveRegMap(BBStarters, false /*After*/, *LIS);
540}
541
542void GCNScheduleDAGMILive::finalizeSchedule() {
543 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
544 LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "All regions recorded, starting actual scheduling.\n"
; } } while (false)
;
1
Assuming 'DebugFlag' is false
2
Loop condition is false. Exiting loop
545
546 LiveIns.resize(Regions.size());
547 Pressure.resize(Regions.size());
548 RescheduleRegions.resize(Regions.size());
549 RegionsWithClusters.resize(Regions.size());
550 RegionsWithHighRP.resize(Regions.size());
551 RescheduleRegions.set();
552 RegionsWithClusters.reset();
553 RegionsWithHighRP.reset();
554
555 if (!Regions.empty())
3
Taking true branch
556 BBLiveInMap = getBBLiveInMap();
557
558 std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
559
560 do {
561 Stage++;
562 RegionIdx = 0;
563 MachineBasicBlock *MBB = nullptr;
4
'MBB' initialized to a null pointer value
564
565 if (Stage > InitialSchedule) {
5
Assuming field 'Stage' is <= InitialSchedule
6
Taking false branch
566 if (!LIS)
567 break;
568
569 // Retry function scheduling if we found resulting occupancy and it is
570 // lower than used for first pass scheduling. This will give more freedom
571 // to schedule low register pressure blocks.
572 // Code is partially copied from MachineSchedulerBase::scheduleRegions().
573
574 if (Stage == UnclusteredReschedule) {
575 if (RescheduleRegions.none())
576 continue;
577 LLVM_DEBUG(dbgs() <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling without clustering.\n"
; } } while (false)
578 "Retrying function scheduling without clustering.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling without clustering.\n"
; } } while (false)
;
579 }
580
581 if (Stage == ClusteredLowOccupancyReschedule) {
582 if (StartingOccupancy <= MinOccupancy)
583 break;
584
585 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
586 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
587 << "Retrying function scheduling with lowest recorded occupancy "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
588 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
;
589
590 S.setTargetOccupancy(MinOccupancy);
591 }
592 }
593
594 if (Stage
6.1
Field 'Stage' is not equal to UnclusteredReschedule
== UnclusteredReschedule)
7
Taking false branch
595 SavedMutations.swap(Mutations);
596
597 for (auto Region : Regions) {
8
Assuming '__begin2' is not equal to '__end2'
598 if ((Stage == UnclusteredReschedule && !RescheduleRegions[RegionIdx]) ||
9
Assuming field 'Stage' is not equal to UnclusteredReschedule
11
Taking false branch
599 (Stage == ClusteredLowOccupancyReschedule &&
10
Assuming field 'Stage' is not equal to ClusteredLowOccupancyReschedule
600 !RegionsWithClusters[RegionIdx] && !RegionsWithHighRP[RegionIdx])) {
601
602 ++RegionIdx;
603 continue;
604 }
605
606 RegionBegin = Region.first;
607 RegionEnd = Region.second;
608
609 if (RegionBegin->getParent() != MBB) {
12
Assuming the condition is false
13
Taking false branch
610 if (MBB) finishBlock();
611 MBB = RegionBegin->getParent();
612 startBlock(MBB);
613 if (Stage == InitialSchedule)
614 computeBlockPressure(MBB);
615 }
616
617 unsigned NumRegionInstrs = std::distance(begin(), end());
618 enterRegion(MBB, begin(), end(), NumRegionInstrs);
619
620 // Skip empty scheduling regions (0 or 1 schedulable instructions).
621 if (begin() == end() || begin() == std::prev(end())) {
14
Taking false branch
622 exitRegion();
623 continue;
624 }
625
626 LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "********** MI Scheduling **********\n"
; } } while (false)
;
15
Assuming 'DebugFlag' is true
16
Assuming the condition is false
17
Taking false branch
18
Loop condition is false. Exiting loop
627 LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
19
Assuming 'DebugFlag' is true
20
Assuming the condition is true
21
Taking true branch
22
Forming reference to null pointer
628 << MBB->getName() << "\n From: " << *begin()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
629 << " To: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
630 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
631 else dbgs() << "End";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
632 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
;
633
634 schedule();
635
636 exitRegion();
637 ++RegionIdx;
638 }
639 finishBlock();
640
641 if (Stage == UnclusteredReschedule)
642 SavedMutations.swap(Mutations);
643 } while (Stage != LastStage);
644}