LLVM 20.0.0git
GCNRegPressure.cpp
Go to the documentation of this file.
1//===- GCNRegPressure.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the GCNRegPressure class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNRegPressure.h"
15#include "AMDGPU.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "machine-scheduler"
21
23 const GCNRPTracker::LiveRegSet &S2) {
24 if (S1.size() != S2.size())
25 return false;
26
27 for (const auto &P : S1) {
28 auto I = S2.find(P.first);
29 if (I == S2.end() || I->second != P.second)
30 return false;
31 }
32 return true;
33}
34
35///////////////////////////////////////////////////////////////////////////////
36// GCNRegPressure
37
38unsigned GCNRegPressure::getRegKind(Register Reg,
39 const MachineRegisterInfo &MRI) {
40 assert(Reg.isVirtual());
41 const auto RC = MRI.getRegClass(Reg);
42 auto STI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
43 return STI->isSGPRClass(RC)
44 ? (STI->getRegSizeInBits(*RC) == 32 ? SGPR32 : SGPR_TUPLE)
45 : STI->isAGPRClass(RC)
46 ? (STI->getRegSizeInBits(*RC) == 32 ? AGPR32 : AGPR_TUPLE)
47 : (STI->getRegSizeInBits(*RC) == 32 ? VGPR32 : VGPR_TUPLE);
48}
49
50void GCNRegPressure::inc(unsigned Reg,
51 LaneBitmask PrevMask,
52 LaneBitmask NewMask,
53 const MachineRegisterInfo &MRI) {
56 return;
57
58 int Sign = 1;
59 if (NewMask < PrevMask) {
60 std::swap(NewMask, PrevMask);
61 Sign = -1;
62 }
63
64 switch (auto Kind = getRegKind(Reg, MRI)) {
65 case SGPR32:
66 case VGPR32:
67 case AGPR32:
68 Value[Kind] += Sign;
69 break;
70
71 case SGPR_TUPLE:
72 case VGPR_TUPLE:
73 case AGPR_TUPLE:
74 assert(PrevMask < NewMask);
75
76 Value[Kind == SGPR_TUPLE ? SGPR32 : Kind == AGPR_TUPLE ? AGPR32 : VGPR32] +=
77 Sign * SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
78
79 if (PrevMask.none()) {
80 assert(NewMask.any());
81 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
82 Value[Kind] +=
83 Sign * TRI->getRegClassWeight(MRI.getRegClass(Reg)).RegWeight;
84 }
85 break;
86
87 default: llvm_unreachable("Unknown register kind");
88 }
89}
90
92 unsigned MaxOccupancy) const {
93 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
94
95 const auto SGPROcc = std::min(MaxOccupancy,
96 ST.getOccupancyWithNumSGPRs(getSGPRNum()));
97 const auto VGPROcc =
98 std::min(MaxOccupancy,
99 ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts())));
100 const auto OtherSGPROcc = std::min(MaxOccupancy,
101 ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
102 const auto OtherVGPROcc =
103 std::min(MaxOccupancy,
104 ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts())));
105
106 const auto Occ = std::min(SGPROcc, VGPROcc);
107 const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);
108
109 // Give first precedence to the better occupancy.
110 if (Occ != OtherOcc)
111 return Occ > OtherOcc;
112
113 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
114 unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
115
116 // SGPR excess pressure conditions
117 unsigned ExcessSGPR = std::max(static_cast<int>(getSGPRNum() - MaxSGPRs), 0);
118 unsigned OtherExcessSGPR =
119 std::max(static_cast<int>(O.getSGPRNum() - MaxSGPRs), 0);
120
121 auto WaveSize = ST.getWavefrontSize();
122 // The number of virtual VGPRs required to handle excess SGPR
123 unsigned VGPRForSGPRSpills = (ExcessSGPR + (WaveSize - 1)) / WaveSize;
124 unsigned OtherVGPRForSGPRSpills =
125 (OtherExcessSGPR + (WaveSize - 1)) / WaveSize;
126
127 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
128
129 // Unified excess pressure conditions, accounting for VGPRs used for SGPR
130 // spills
131 unsigned ExcessVGPR =
132 std::max(static_cast<int>(getVGPRNum(ST.hasGFX90AInsts()) +
133 VGPRForSGPRSpills - MaxVGPRs),
134 0);
135 unsigned OtherExcessVGPR =
136 std::max(static_cast<int>(O.getVGPRNum(ST.hasGFX90AInsts()) +
137 OtherVGPRForSGPRSpills - MaxVGPRs),
138 0);
139 // Arch VGPR excess pressure conditions, accounting for VGPRs used for SGPR
140 // spills
141 unsigned ExcessArchVGPR = std::max(
142 static_cast<int>(getVGPRNum(false) + VGPRForSGPRSpills - MaxArchVGPRs),
143 0);
144 unsigned OtherExcessArchVGPR =
145 std::max(static_cast<int>(O.getVGPRNum(false) + OtherVGPRForSGPRSpills -
146 MaxArchVGPRs),
147 0);
148 // AGPR excess pressure conditions
149 unsigned ExcessAGPR = std::max(
150 static_cast<int>(ST.hasGFX90AInsts() ? (getAGPRNum() - MaxArchVGPRs)
151 : (getAGPRNum() - MaxVGPRs)),
152 0);
153 unsigned OtherExcessAGPR = std::max(
154 static_cast<int>(ST.hasGFX90AInsts() ? (O.getAGPRNum() - MaxArchVGPRs)
155 : (O.getAGPRNum() - MaxVGPRs)),
156 0);
157
158 bool ExcessRP = ExcessSGPR || ExcessVGPR || ExcessArchVGPR || ExcessAGPR;
159 bool OtherExcessRP = OtherExcessSGPR || OtherExcessVGPR ||
160 OtherExcessArchVGPR || OtherExcessAGPR;
161
162 // Give second precedence to the reduced number of spills to hold the register
163 // pressure.
164 if (ExcessRP || OtherExcessRP) {
165 // The difference in excess VGPR pressure, after including VGPRs used for
166 // SGPR spills
167 int VGPRDiff = ((OtherExcessVGPR + OtherExcessArchVGPR + OtherExcessAGPR) -
168 (ExcessVGPR + ExcessArchVGPR + ExcessAGPR));
169
170 int SGPRDiff = OtherExcessSGPR - ExcessSGPR;
171
172 if (VGPRDiff != 0)
173 return VGPRDiff > 0;
174 if (SGPRDiff != 0) {
175 unsigned PureExcessVGPR =
176 std::max(static_cast<int>(getVGPRNum(ST.hasGFX90AInsts()) - MaxVGPRs),
177 0) +
178 std::max(static_cast<int>(getVGPRNum(false) - MaxArchVGPRs), 0);
179 unsigned OtherPureExcessVGPR =
180 std::max(
181 static_cast<int>(O.getVGPRNum(ST.hasGFX90AInsts()) - MaxVGPRs),
182 0) +
183 std::max(static_cast<int>(O.getVGPRNum(false) - MaxArchVGPRs), 0);
184
185 // If we have a special case where there is a tie in excess VGPR, but one
186 // of the pressures has VGPR usage from SGPR spills, prefer the pressure
187 // with SGPR spills.
188 if (PureExcessVGPR != OtherPureExcessVGPR)
189 return SGPRDiff < 0;
190 // If both pressures have the same excess pressure before and after
191 // accounting for SGPR spills, prefer fewer SGPR spills.
192 return SGPRDiff > 0;
193 }
194 }
195
196 bool SGPRImportant = SGPROcc < VGPROcc;
197 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
198
199 // If both pressures disagree on what is more important compare vgprs.
200 if (SGPRImportant != OtherSGPRImportant) {
201 SGPRImportant = false;
202 }
203
204 // Give third precedence to lower register tuple pressure.
205 bool SGPRFirst = SGPRImportant;
206 for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
207 if (SGPRFirst) {
208 auto SW = getSGPRTuplesWeight();
209 auto OtherSW = O.getSGPRTuplesWeight();
210 if (SW != OtherSW)
211 return SW < OtherSW;
212 } else {
213 auto VW = getVGPRTuplesWeight();
214 auto OtherVW = O.getVGPRTuplesWeight();
215 if (VW != OtherVW)
216 return VW < OtherVW;
217 }
218 }
219
220 // Give final precedence to lower general RP.
221 return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
222 (getVGPRNum(ST.hasGFX90AInsts()) <
223 O.getVGPRNum(ST.hasGFX90AInsts()));
224}
225
227 return Printable([&RP, ST](raw_ostream &OS) {
228 OS << "VGPRs: " << RP.Value[GCNRegPressure::VGPR32] << ' '
229 << "AGPRs: " << RP.getAGPRNum();
230 if (ST)
231 OS << "(O"
232 << ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()))
233 << ')';
234 OS << ", SGPRs: " << RP.getSGPRNum();
235 if (ST)
236 OS << "(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) << ')';
237 OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()
238 << ", LSGPR WT: " << RP.getSGPRTuplesWeight();
239 if (ST)
240 OS << " -> Occ: " << RP.getOccupancy(*ST);
241 OS << '\n';
242 });
243}
244
246 const MachineRegisterInfo &MRI) {
247 assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());
248
249 // We don't rely on read-undef flag because in case of tentative schedule
250 // tracking it isn't set correctly yet. This works correctly however since
251 // use mask has been tracked before using LIS.
252 return MO.getSubReg() == 0 ?
253 MRI.getMaxLaneMaskForVReg(MO.getReg()) :
254 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
255}
256
257static void
259 const MachineInstr &MI, const LiveIntervals &LIS,
260 const MachineRegisterInfo &MRI) {
261 SlotIndex InstrSI;
262 for (const auto &MO : MI.operands()) {
263 if (!MO.isReg() || !MO.getReg().isVirtual())
264 continue;
265 if (!MO.isUse() || !MO.readsReg())
266 continue;
267
268 Register Reg = MO.getReg();
269 if (llvm::any_of(RegMaskPairs, [Reg](const RegisterMaskPair &RM) {
270 return RM.RegUnit == Reg;
271 }))
272 continue;
273
274 LaneBitmask UseMask;
275 auto &LI = LIS.getInterval(Reg);
276 if (!LI.hasSubRanges())
277 UseMask = MRI.getMaxLaneMaskForVReg(Reg);
278 else {
279 // For a tentative schedule LIS isn't updated yet but livemask should
280 // remain the same on any schedule. Subreg defs can be reordered but they
281 // all must dominate uses anyway.
282 if (!InstrSI)
283 InstrSI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
284 UseMask = getLiveLaneMask(LI, InstrSI, MRI);
285 }
286
287 RegMaskPairs.emplace_back(Reg, UseMask);
288 }
289}
290
291///////////////////////////////////////////////////////////////////////////////
292// GCNRPTracker
293
295 const LiveIntervals &LIS,
296 const MachineRegisterInfo &MRI) {
297 return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI);
298}
299
301 const MachineRegisterInfo &MRI) {
302 LaneBitmask LiveMask;
303 if (LI.hasSubRanges()) {
304 for (const auto &S : LI.subranges())
305 if (S.liveAt(SI)) {
306 LiveMask |= S.LaneMask;
307 assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));
308 }
309 } else if (LI.liveAt(SI)) {
310 LiveMask = MRI.getMaxLaneMaskForVReg(LI.reg());
311 }
312 return LiveMask;
313}
314
316 const LiveIntervals &LIS,
317 const MachineRegisterInfo &MRI) {
319 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
320 auto Reg = Register::index2VirtReg(I);
321 if (!LIS.hasInterval(Reg))
322 continue;
323 auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
324 if (LiveMask.any())
325 LiveRegs[Reg] = LiveMask;
326 }
327 return LiveRegs;
328}
329
331 const LiveRegSet *LiveRegsCopy,
332 bool After) {
333 const MachineFunction &MF = *MI.getMF();
334 MRI = &MF.getRegInfo();
335 if (LiveRegsCopy) {
336 if (&LiveRegs != LiveRegsCopy)
337 LiveRegs = *LiveRegsCopy;
338 } else {
341 }
342
344}
345
346////////////////////////////////////////////////////////////////////////////////
347// GCNUpwardRPTracker
348
350 const LiveRegSet &LiveRegs_) {
351 MRI = &MRI_;
352 LiveRegs = LiveRegs_;
353 LastTrackedMI = nullptr;
354 MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);
355}
356
358 assert(MRI && "call reset first");
359
360 LastTrackedMI = &MI;
361
362 if (MI.isDebugInstr())
363 return;
364
365 // Kill all defs.
366 GCNRegPressure DefPressure, ECDefPressure;
367 bool HasECDefs = false;
368 for (const MachineOperand &MO : MI.all_defs()) {
369 if (!MO.getReg().isVirtual())
370 continue;
371
372 Register Reg = MO.getReg();
373 LaneBitmask DefMask = getDefRegMask(MO, *MRI);
374
375 // Treat a def as fully live at the moment of definition: keep a record.
376 if (MO.isEarlyClobber()) {
377 ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
378 HasECDefs = true;
379 } else
380 DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
381
382 auto I = LiveRegs.find(Reg);
383 if (I == LiveRegs.end())
384 continue;
385
386 LaneBitmask &LiveMask = I->second;
387 LaneBitmask PrevMask = LiveMask;
388 LiveMask &= ~DefMask;
389 CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
390 if (LiveMask.none())
392 }
393
394 // Update MaxPressure with defs pressure.
395 DefPressure += CurPressure;
396 if (HasECDefs)
397 DefPressure += ECDefPressure;
398 MaxPressure = max(DefPressure, MaxPressure);
399
400 // Make uses alive.
402 collectVirtualRegUses(RegUses, MI, LIS, *MRI);
403 for (const RegisterMaskPair &U : RegUses) {
404 LaneBitmask &LiveMask = LiveRegs[U.RegUnit];
405 LaneBitmask PrevMask = LiveMask;
406 LiveMask |= U.LaneMask;
407 CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI);
408 }
409
410 // Update MaxPressure with uses plus early-clobber defs pressure.
411 MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)
413
415}
416
417////////////////////////////////////////////////////////////////////////////////
418// GCNDownwardRPTracker
419
421 const LiveRegSet *LiveRegsCopy) {
422 MRI = &MI.getParent()->getParent()->getRegInfo();
423 LastTrackedMI = nullptr;
424 MBBEnd = MI.getParent()->end();
425 NextMI = &MI;
426 NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
427 if (NextMI == MBBEnd)
428 return false;
429 GCNRPTracker::reset(*NextMI, LiveRegsCopy, false);
430 return true;
431}
432
434 assert(MRI && "call reset first");
435 if (!LastTrackedMI)
436 return NextMI == MBBEnd;
437
438 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
439
440 SlotIndex SI = NextMI == MBBEnd
441 ? LIS.getInstructionIndex(*LastTrackedMI).getDeadSlot()
443 assert(SI.isValid());
444
445 // Remove dead registers or mask bits.
446 SmallSet<Register, 8> SeenRegs;
447 for (auto &MO : LastTrackedMI->operands()) {
448 if (!MO.isReg() || !MO.getReg().isVirtual())
449 continue;
450 if (MO.isUse() && !MO.readsReg())
451 continue;
452 if (!SeenRegs.insert(MO.getReg()).second)
453 continue;
454 const LiveInterval &LI = LIS.getInterval(MO.getReg());
455 if (LI.hasSubRanges()) {
456 auto It = LiveRegs.end();
457 for (const auto &S : LI.subranges()) {
458 if (!S.liveAt(SI)) {
459 if (It == LiveRegs.end()) {
460 It = LiveRegs.find(MO.getReg());
461 if (It == LiveRegs.end())
462 llvm_unreachable("register isn't live");
463 }
464 auto PrevMask = It->second;
465 It->second &= ~S.LaneMask;
466 CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);
467 }
468 }
469 if (It != LiveRegs.end() && It->second.none())
470 LiveRegs.erase(It);
471 } else if (!LI.liveAt(SI)) {
472 auto It = LiveRegs.find(MO.getReg());
473 if (It == LiveRegs.end())
474 llvm_unreachable("register isn't live");
475 CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);
476 LiveRegs.erase(It);
477 }
478 }
479
481
482 LastTrackedMI = nullptr;
483
484 return NextMI == MBBEnd;
485}
486
488 LastTrackedMI = &*NextMI++;
489 NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
490
491 // Add new registers or mask bits.
492 for (const auto &MO : LastTrackedMI->all_defs()) {
493 Register Reg = MO.getReg();
494 if (!Reg.isVirtual())
495 continue;
496 auto &LiveMask = LiveRegs[Reg];
497 auto PrevMask = LiveMask;
498 LiveMask |= getDefRegMask(MO, *MRI);
499 CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
500 }
501
503}
504
506 if (NextMI == MBBEnd)
507 return false;
510 return true;
511}
512
514 while (NextMI != End)
515 if (!advance()) return false;
516 return true;
517}
518
521 const LiveRegSet *LiveRegsCopy) {
522 reset(*Begin, LiveRegsCopy);
523 return advance(End);
524}
525
527 const GCNRPTracker::LiveRegSet &TrackedLR,
528 const TargetRegisterInfo *TRI, StringRef Pfx) {
529 return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {
530 for (auto const &P : TrackedLR) {
531 auto I = LISLR.find(P.first);
532 if (I == LISLR.end()) {
533 OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
534 << " isn't found in LIS reported set\n";
535 } else if (I->second != P.second) {
536 OS << Pfx << printReg(P.first, TRI)
537 << " masks doesn't match: LIS reported " << PrintLaneMask(I->second)
538 << ", tracked " << PrintLaneMask(P.second) << '\n';
539 }
540 }
541 for (auto const &P : LISLR) {
542 auto I = TrackedLR.find(P.first);
543 if (I == TrackedLR.end()) {
544 OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
545 << " isn't found in tracked set\n";
546 }
547 }
548 });
549}
550
552 const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();
553 const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);
554 const auto &TrackedLR = LiveRegs;
555
556 if (!isEqual(LISLR, TrackedLR)) {
557 dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
558 " LIS reported livesets mismatch:\n"
559 << print(LISLR, *MRI);
560 reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());
561 return false;
562 }
563
564 auto LISPressure = getRegPressure(*MRI, LISLR);
565 if (LISPressure != CurPressure) {
566 dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
567 << print(CurPressure) << "LIS rpt: " << print(LISPressure);
568 return false;
569 }
570 return true;
571}
572
574 const MachineRegisterInfo &MRI) {
575 return Printable([&LiveRegs, &MRI](raw_ostream &OS) {
576 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
577 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
578 Register Reg = Register::index2VirtReg(I);
579 auto It = LiveRegs.find(Reg);
580 if (It != LiveRegs.end() && It->second.any())
581 OS << ' ' << printVRegOrUnit(Reg, TRI) << ':'
582 << PrintLaneMask(It->second);
583 }
584 OS << '\n';
585 });
586}
587
588void GCNRegPressure::dump() const { dbgs() << print(*this); }
589
591 "amdgpu-print-rp-downward",
592 cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"),
593 cl::init(false), cl::Hidden);
594
597
598INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)
599
600// Return lanemask of Reg's subregs that are live-through at [Begin, End] and
601// are fully covered by Mask.
602static LaneBitmask
604 Register Reg, SlotIndex Begin, SlotIndex End,
605 LaneBitmask Mask = LaneBitmask::getAll()) {
606
607 auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {
608 auto *Segment = LR.getSegmentContaining(Begin);
609 return Segment && Segment->contains(End);
610 };
611
612 LaneBitmask LiveThroughMask;
613 const LiveInterval &LI = LIS.getInterval(Reg);
614 if (LI.hasSubRanges()) {
615 for (auto &SR : LI.subranges()) {
616 if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))
617 LiveThroughMask |= SR.LaneMask;
618 }
619 } else {
620 LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);
621 if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))
622 LiveThroughMask = RegMask;
623 }
624
625 return LiveThroughMask;
626}
627
629 const MachineRegisterInfo &MRI = MF.getRegInfo();
630 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
631 const LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
632
633 auto &OS = dbgs();
634
635// Leading spaces are important for YAML syntax.
636#define PFX " "
637
638 OS << "---\nname: " << MF.getName() << "\nbody: |\n";
639
640 auto printRP = [](const GCNRegPressure &RP) {
641 return Printable([&RP](raw_ostream &OS) {
642 OS << format(PFX " %-5d", RP.getSGPRNum())
643 << format(" %-5d", RP.getVGPRNum(false));
644 });
645 };
646
647 auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,
648 const GCNRPTracker::LiveRegSet &LISLR) {
649 if (LISLR != TrackedLR) {
650 OS << PFX " mis LIS: " << llvm::print(LISLR, MRI)
651 << reportMismatch(LISLR, TrackedLR, TRI, PFX " ");
652 }
653 };
654
655 // Register pressure before and at an instruction (in program order).
657
658 for (auto &MBB : MF) {
659 RP.clear();
660 RP.reserve(MBB.size());
661
662 OS << PFX;
664 OS << ":\n";
665
666 SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(&MBB);
667 SlotIndex MBBEndSlot = LIS.getSlotIndexes()->getMBBEndIdx(&MBB);
668
669 GCNRPTracker::LiveRegSet LiveIn, LiveOut;
670 GCNRegPressure RPAtMBBEnd;
671
672 if (UseDownwardTracker) {
673 if (MBB.empty()) {
674 LiveIn = LiveOut = getLiveRegs(MBBStartSlot, LIS, MRI);
675 RPAtMBBEnd = getRegPressure(MRI, LiveIn);
676 } else {
677 GCNDownwardRPTracker RPT(LIS);
678 RPT.reset(MBB.front());
679
680 LiveIn = RPT.getLiveRegs();
681
682 while (!RPT.advanceBeforeNext()) {
683 GCNRegPressure RPBeforeMI = RPT.getPressure();
684 RPT.advanceToNext();
685 RP.emplace_back(RPBeforeMI, RPT.getPressure());
686 }
687
688 LiveOut = RPT.getLiveRegs();
689 RPAtMBBEnd = RPT.getPressure();
690 }
691 } else {
692 GCNUpwardRPTracker RPT(LIS);
693 RPT.reset(MRI, MBBEndSlot);
694
695 LiveOut = RPT.getLiveRegs();
696 RPAtMBBEnd = RPT.getPressure();
697
698 for (auto &MI : reverse(MBB)) {
699 RPT.resetMaxPressure();
700 RPT.recede(MI);
701 if (!MI.isDebugInstr())
702 RP.emplace_back(RPT.getPressure(), RPT.getMaxPressure());
703 }
704
705 LiveIn = RPT.getLiveRegs();
706 }
707
708 OS << PFX " Live-in: " << llvm::print(LiveIn, MRI);
710 ReportLISMismatchIfAny(LiveIn, getLiveRegs(MBBStartSlot, LIS, MRI));
711
712 OS << PFX " SGPR VGPR\n";
713 int I = 0;
714 for (auto &MI : MBB) {
715 if (!MI.isDebugInstr()) {
716 auto &[RPBeforeInstr, RPAtInstr] =
717 RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];
718 ++I;
719 OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << " ";
720 } else
721 OS << PFX " ";
722 MI.print(OS);
723 }
724 OS << printRP(RPAtMBBEnd) << '\n';
725
726 OS << PFX " Live-out:" << llvm::print(LiveOut, MRI);
728 ReportLISMismatchIfAny(LiveOut, getLiveRegs(MBBEndSlot, LIS, MRI));
729
730 GCNRPTracker::LiveRegSet LiveThrough;
731 for (auto [Reg, Mask] : LiveIn) {
732 LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Reg);
733 if (MaskIntersection.any()) {
735 MRI, LIS, Reg, MBBStartSlot, MBBEndSlot, MaskIntersection);
736 if (LTMask.any())
737 LiveThrough[Reg] = LTMask;
738 }
739 }
740 OS << PFX " Live-thr:" << llvm::print(LiveThrough, MRI);
741 OS << printRP(getRegPressure(MRI, LiveThrough)) << '\n';
742 }
743 OS << "...\n";
744 return false;
745
746#undef PFX
747}
unsigned const MachineRegisterInfo * MRI
aarch64 promote const
static const LLT S1
MachineBasicBlock & MBB
bool End
Definition: ELF_riscv.cpp:480
#define PFX
static cl::opt< bool > UseDownwardTracker("amdgpu-print-rp-downward", cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"), cl::init(false), cl::Hidden)
static void collectVirtualRegUses(SmallVectorImpl< RegisterMaskPair > &RegMaskPairs, const MachineInstr &MI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
static LaneBitmask getDefRegMask(const MachineOperand &MO, const MachineRegisterInfo &MRI)
static LaneBitmask getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS, Register Reg, SlotIndex Begin, SlotIndex End, LaneBitmask Mask=LaneBitmask::getAll())
This file defines the GCNRegPressure class, which tracks registry pressure by bookkeeping number of S...
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
#define P(N)
if(PassOpts->AAPipeline)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
bool erase(const KeyT &Val)
Definition: DenseMap.h:336
unsigned size() const
Definition: DenseMap.h:99
iterator end()
Definition: DenseMap.h:84
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
GCNRegPressure getPressure() const
const decltype(LiveRegs) & getLiveRegs() const
const MachineInstr * LastTrackedMI
GCNRegPressure CurPressure
GCNRegPressure MaxPressure
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, bool After)
const MachineRegisterInfo * MRI
const LiveIntervals & LIS
void recede(const MachineInstr &MI)
void reset(const MachineRegisterInfo &MRI_, const LiveRegSet &LiveRegs_)
const GCNRegPressure & getMaxPressure() const
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:687
Register reg() const
Definition: LiveInterval.h:718
bool hasSubRanges() const
Returns true if subregister liveness information is available.
Definition: LiveInterval.h:810
iterator_range< subrange_iterator > subranges()
Definition: LiveInterval.h:782
bool hasInterval(Register Reg) const
SlotIndexes * getSlotIndexes() const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
This class represents the liveness of a register, stack slot, etc.
Definition: LiveInterval.h:157
bool liveAt(SlotIndex index) const
Definition: LiveInterval.h:401
void printName(raw_ostream &os, unsigned printNameFlags=PrintNameIr, ModuleSlotTracker *moduleSlotTracker=nullptr) const
Print the basic block's name as:
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Representation of each machine instruction.
Definition: MachineInstr.h:69
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:685
iterator_range< filtered_mop_iterator > all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
Definition: MachineInstr.h:756
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterInfo * getTargetRegisterInfo() const
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
Definition: Pass.cpp:130
Simple wrapper around std::function<void(raw_ostream&)>.
Definition: Printable.h:38
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition: Register.h:84
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static unsigned getNumCoveredRegs(LaneBitmask LM)
static bool isSGPRClass(const TargetRegisterClass *RC)
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:65
SlotIndex getDeadSlot() const
Returns the dead def kill slot for the current instruction.
Definition: SlotIndexes.h:242
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
Definition: SlotIndexes.h:224
SlotIndex getMBBEndIdx(unsigned Num) const
Returns the last index in the given basic block number.
Definition: SlotIndexes.h:470
SlotIndex getMBBStartIdx(unsigned Num) const
Returns the first index in the given basic block number.
Definition: SlotIndexes.h:460
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:587
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:951
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
Definition: Value.h:74
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, const LiveIntervals &LIS)
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI, const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
char & GCNRegPressurePrinterID
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR, const GCNRPTracker::LiveRegSet &TrackedL, const TargetRegisterInfo *TRI, StringRef Pfx=" ")
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getVGPRTuplesWeight() const
unsigned getVGPRNum(bool UnifiedVGPRFile) const
void inc(unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask, const MachineRegisterInfo &MRI)
unsigned getAGPRNum() const
unsigned getSGPRNum() const
unsigned getSGPRTuplesWeight() const
friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST)
bool less(const MachineFunction &MF, const GCNRegPressure &O, unsigned MaxOccupancy=std::numeric_limits< unsigned >::max()) const
Compares this GCNRegpressure to O, returning true if this is less.
constexpr bool none() const
Definition: LaneBitmask.h:52
constexpr bool any() const
Definition: LaneBitmask.h:53
static constexpr LaneBitmask getNone()
Definition: LaneBitmask.h:81