LLVM 24.0.0git
GCNPreRAAntiHints.cpp
Go to the documentation of this file.
1//===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert register allocation anti-hints.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNPreRAAntiHints.h"
15#include "GCNSubtarget.h"
16#include "SIInstrInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/STLExtras.h"
23
24using namespace llvm;
25using namespace llvm::AMDGPU;
26
27#define DEBUG_TYPE "amdgpu-anti-hints"
28
30
31namespace {
32
33// Classify the MI into a HazardClassMask.
34HazardClassMask getInstHazardClass(const MachineInstr &MI,
35 const HazardContext &Ctx) {
36 const SIInstrInfo &TII = *Ctx.TII;
38
39 if (TII.isLDSDMA(MI))
41 else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
42 Mask = HC::WMMA;
43 else if (TII.isMFMA(MI))
44 Mask = HC::MFMA;
45 else if (SIInstrInfo::isTRANS(MI))
46 Mask = HC::TRANS;
47 else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
48 Mask = HC::VALU;
49 else if (TII.isDS(MI))
50 Mask = HC::DS;
51 else if (TII.isVMEM(MI))
52 Mask = HC::VMEM;
53 else if (TII.isSMRD(MI))
54 Mask = HC::SMEM;
55 else if (TII.isEXP(MI))
56 Mask = HC::EXP;
57 else if (SIInstrInfo::isSALU(MI))
58 Mask = HC::SALU;
59 else if (MI.isCopy() && MI.getOperand(0).getReg().isVirtual() &&
60 Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MI.getOperand(0).getReg())))
61 Mask = HC::VALU;
62 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
63 Mask |= HC::RawVALU;
64 if (MI.mayLoad())
65 Mask |= HC::Load;
66 for (const MachineOperand &MO : MI.defs()) {
67 if (MO.isReg() && MO.getReg().isVirtual() &&
68 Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO.getReg()))) {
69 Mask |= HC::WritesVGPR;
70 break;
71 }
72 }
73
74 return Mask;
75}
76
77bool isVirtualVGPR(const HazardContext &Ctx, const MachineOperand &MO) {
78 if (!MO.isReg() || !MO.getReg().isVirtual())
79 return false;
80 return Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO.getReg()));
81}
82
83void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
84 const HazardContext &Ctx,
86 const SIInstrInfo &TII = *Ctx.TII;
87 switch (Op) {
89 break;
91 for (const MachineOperand &MO : MI.all_defs()) {
92 if (isVirtualVGPR(Ctx, MO))
93 Out.push_back(MO.getReg());
94 }
95 break;
97 if (const MachineOperand *MO =
98 TII.getNamedOperand(MI, AMDGPU::OpName::src0)) {
99 if (isVirtualVGPR(Ctx, *MO))
100 Out.push_back(MO->getReg());
101 }
102 break;
104 if (const MachineOperand *MO =
105 TII.getNamedOperand(MI, AMDGPU::OpName::src1)) {
106 if (isVirtualVGPR(Ctx, *MO))
107 Out.push_back(MO->getReg());
108 }
109 break;
111 if (const MachineOperand *MO =
112 TII.getNamedOperand(MI, AMDGPU::OpName::src2)) {
113 if (isVirtualVGPR(Ctx, *MO))
114 Out.push_back(MO->getReg());
115 }
116 break;
118 for (AMDGPU::OpName Name : {AMDGPU::OpName::src0, AMDGPU::OpName::src1}) {
119 if (const MachineOperand *MO = TII.getNamedOperand(MI, Name)) {
120 if (isVirtualVGPR(Ctx, *MO))
121 Out.push_back(MO->getReg());
122 }
123 }
124 break;
126 if (const MachineOperand *MO =
127 TII.getNamedOperand(MI, AMDGPU::OpName::idx)) {
128 if (isVirtualVGPR(Ctx, *MO))
129 Out.push_back(MO->getReg());
130 }
131 break;
133 if (const MachineOperand *MO =
134 TII.getNamedOperand(MI, AMDGPU::OpName::vaddr)) {
135 if (isVirtualVGPR(Ctx, *MO))
136 Out.push_back(MO->getReg());
137 }
138 break;
140 for (AMDGPU::OpName Name :
141 {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
142 if (const MachineOperand *MO = TII.getNamedOperand(MI, Name)) {
143 if (isVirtualVGPR(Ctx, *MO))
144 Out.push_back(MO->getReg());
145 }
146 }
147 break;
149 for (const MachineOperand &MO : MI.all_uses()) {
150 if (isVirtualVGPR(Ctx, MO))
151 Out.push_back(MO.getReg());
152 }
153 break;
154 }
155}
156
157enum class MFMAHazardKind { RAW, WAW, WAR };
158
159// MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait
160// states.
161unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
162 HazardClassMask ReaderClass, const HazardContext &Ctx) {
163 const SIInstrInfo &TII = *Ctx.TII;
164 const GCNSubtarget &ST = *Ctx.ST;
165 const int NumPasses = Ctx.SchedModel->computeInstrLatency(&MFMA);
166 const bool IsDGEMM = SIInstrInfo::isDGEMM(MFMA.getOpcode());
167 const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP);
168
169 auto GFX940NPass = [&]() -> unsigned {
170 return TII.isXDL(MFMA)
171 ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
172 : NumPasses + 2;
173 };
174 auto SMFMANPass = [&]() -> unsigned {
175 switch (NumPasses) {
176 case 2:
177 return 5;
178 case 8:
179 return 11;
180 case 16:
181 return 19;
182 default:
183 return 0;
184 }
185 };
186
187 switch (Kind) {
188 case MFMAHazardKind::RAW:
189 if (IsDGEMM) {
190 switch (NumPasses) {
191 case 4:
192 return Mem ? 9 : 6;
193 case 8:
194 case 16:
195 return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
196 default:
197 return 0;
198 }
199 }
200 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
201
202 case MFMAHazardKind::WAW:
203 if (IsDGEMM) {
204 switch (NumPasses) {
205 case 4:
206 return 6;
207 case 8:
208 case 16:
209 return 11;
210 default:
211 return 0;
212 }
213 }
214 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
215
216 case MFMAHazardKind::WAR:
217 switch (NumPasses) {
218 case 2:
219 return 1;
220 case 4:
221 return 3;
222 case 8:
223 return 7;
224 case 16:
225 return 15;
226 default:
227 return 15;
228 }
229 }
230 return 0;
231}
232
233unsigned mfmaWawWindow(const MachineInstr &P, HazardClassMask,
234 const HazardContext &Ctx) {
235 return mfmaWaitStates(P, MFMAHazardKind::WAW, HC::None, Ctx);
236}
237unsigned mfmaWarWindow(const MachineInstr &P, HazardClassMask,
238 const HazardContext &Ctx) {
239 return mfmaWaitStates(P, MFMAHazardKind::WAR, HC::None, Ctx);
240}
241unsigned addrAntiHintWindow(const MachineInstr &, HazardClassMask,
242 const HazardContext &Ctx) {
243 return Ctx.ST->getCLOpts().addr_anti_hint_window;
244}
245unsigned vaVdstLookbackWindow(const MachineInstr &, HazardClassMask,
246 const HazardContext &Ctx) {
247 return Ctx.ST->getCLOpts().va_vdst_lookback_window;
248}
249
250constexpr unsigned NumWMMAHazardCategories = 7;
251unsigned wmmaHazardCategory(const MachineInstr &Producer,
252 const HazardContext &Ctx) {
253 const bool IsSWMMAC = SIInstrInfo::isSWMMAC(Producer);
254 const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA();
255 switch (Ctx.SchedModel->computeInstrLatency(&Producer)) {
256 case 4:
257 return 6;
258 case 8:
259 return IsSWMMAC ? 2 : 0;
260 case 16:
261 return LowestRate ? 4 : (IsSWMMAC ? 3 : 1);
262 case 32:
263 return 5;
264 default:
265 return NumWMMAHazardCategories;
266 }
267}
268
269// WMMA/SWMMAC co-exec window.
270unsigned wmmaCoexecWindow(const MachineInstr &Producer,
271 HazardClassMask ConsumerClass,
272 const HazardContext &Ctx) {
273 const unsigned Category = wmmaHazardCategory(Producer, Ctx);
274 if (Category >= NumWMMAHazardCategories)
275 return 0;
276
277 static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
278 static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
279 return (ConsumerClass & HC::WMMA) ? WMMAWaitStates[Category]
280 : VALUWaitStates[Category];
281}
282
283unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
284 HazardClassMask ReaderClass,
285 const HazardContext &Ctx) {
286 return mfmaWaitStates(Producer, MFMAHazardKind::RAW, ReaderClass, Ctx);
287}
288
289bool hasMFMAHazard(const HazardContext &Ctx) {
290 return Ctx.ST->hasGFX90AInsts();
291}
292
293bool hasWMMACoexecHazard(const HazardContext &Ctx) {
294 return Ctx.ST->hasWMMACoexecutionHazards();
295}
296
297bool hasTransCoexecHazard(const HazardContext &Ctx) {
298 return Ctx.ST->hasTransCoexecutionHazard();
299}
300
301bool hasGFX1250Insts(const HazardContext &Ctx) {
302 return Ctx.ST->hasGFX1250Insts();
303}
304
305bool ruleSelected(const HazardContext &Ctx, AntiHintRule Rule) {
306 ArrayRef<AntiHintRule> Selection = Ctx.ST->getCLOpts().anti_hints_rules;
307 // No -amdgpu-anti-hints-rules on the command line selects every rule.
309 return true;
310 return is_contained(Selection, Rule);
311}
312
313bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
314 return hasMFMAHazard(Ctx) && ruleSelected(Ctx, AntiHintRule::MFMAWAW);
315}
316
317bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
318 return hasMFMAHazard(Ctx) && ruleSelected(Ctx, AntiHintRule::MFMAWAR);
319}
320
321bool isWMMAWARABRuleEnabled(const HazardContext &Ctx) {
322 return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, AntiHintRule::WMMAWARAB);
323}
324
325bool isSWMMACWARIndexRuleEnabled(const HazardContext &Ctx) {
326 return hasWMMACoexecHazard(Ctx) &&
327 ruleSelected(Ctx, AntiHintRule::SWMMACWARIndex);
328}
329
330bool isWMMAWAWRuleEnabled(const HazardContext &Ctx) {
331 return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, AntiHintRule::WMMAWAW);
332}
333
334bool isTransWARRuleEnabled(const HazardContext &Ctx) {
335 return hasTransCoexecHazard(Ctx) && ruleSelected(Ctx, AntiHintRule::TransWAR);
336}
337
338bool isAddrWARRuleEnabled(const HazardContext &Ctx) {
339 return Ctx.ST->getCLOpts().anti_hints_for_addr && hasGFX1250Insts(Ctx) &&
340 ruleSelected(Ctx, AntiHintRule::MemAddrWAR);
341}
342
343bool isVAVdstWARRuleEnabled(const HazardContext &Ctx) {
344 return Ctx.ST->getCLOpts().anti_hints_for_va_vdst && hasGFX1250Insts(Ctx) &&
345 ruleSelected(Ctx, AntiHintRule::VAVdstWAR);
346}
347
348bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
349 return Ctx.TII->isXDL(MI);
350}
351
352bool isXDLWMMA(const MachineInstr &MI, const HazardContext &Ctx) {
353 return Ctx.TII->isXDLWMMA(MI);
354}
355
356bool isXDLSWMMAC(const MachineInstr &MI, const HazardContext &Ctx) {
357 return Ctx.TII->isXDLWMMA(MI) && SIInstrInfo::isSWMMAC(MI);
358}
359
360bool isDSorVMEMLoad(const MachineInstr &MI, const HazardContext &Ctx) {
361 return MI.mayLoad() && (Ctx.TII->isDS(MI) || Ctx.TII->isVMEM(MI));
362}
363
364unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
365 const HazardContext &Ctx) {
366 if (CT.Window.Fn)
367 return CT.Window.Fn(MI, CT.Side.Match.AnyOf, Ctx);
368 return CT.Window.WindowLength;
369}
370
371// Build the anti-hints rules.
372class HazardRuleSet {
374
375public:
376 class RuleBuilder {
377
378 HazardRuleSet &S;
379 unsigned Idx;
380 HazardAntiHintRule &rule() const { return S.Rules[Idx]; }
381
382 public:
383 RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {}
384
385 RuleBuilder &producer(ClassMatch M, HazardOperand Op,
386 InstPredicate Predicate = nullptr) {
387 rule().Producer = {M, Op, Predicate};
388 return *this;
389 }
390
391 RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) {
392 rule().AdvanceForRawWindow = AdvanceForRawWindow;
393 return *this;
394 }
395
396 RuleBuilder &
397 consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
398 HazardClassMask CountMask = 0,
400 InstPredicate Predicate = nullptr) {
401 rule().Consumers.push_back(
402 {{M, Op, Predicate}, Window, CountMask, Direction});
403 return *this;
404 }
405
406 RuleBuilder &enabledIf(RulePredicate Predicate) {
407 rule().Predicate = Predicate;
408 return *this;
409 }
410
411 RuleBuilder &lifetime(Lifetime Life) {
412 rule().Life = Life;
413 return *this;
414 }
415 };
416
417 RuleBuilder addRule() {
418 Rules.emplace_back();
419 return RuleBuilder(*this, Rules.size() - 1);
420 }
421
422 SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); }
423};
424
425// Here the anti-hints rules are inserted.
426SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
427 using HO = HazardOperand;
428 HazardRuleSet S;
429
430 // MFMA relevent windows and consumers
431 const WindowSpec MfmaWawWindow{0, mfmaWawWindow};
432 const WindowSpec MfmaWarWindow{0, mfmaWarWindow};
433 const ClassMatch MfmaConsumers = {HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
434 HC::EXP};
435
436 // WMMA relevent windows and consumers
437 const WindowSpec WmmaCoexecWindow{0, wmmaCoexecWindow};
438 const ClassMatch WmmaCoexecConsumers = {/*AnyOf=*/HC::VALU | HC::TRANS,
439 /*AllOf=*/HC::None,
440 /*NoneOf=*/HC::LDSDMA};
441 const HazardClassMask CoexecCounters = HC::VALU | HC::TRANS | HC::WMMA;
442
443 // MFMA WAW rules
444 S.addRule()
445 .enabledIf(isMFMAWAWRuleEnabled)
446 .producer({HC::MFMA}, HO::Def)
447 .rawCredit(mfmaReaderRawWindow)
448 .consumer(MfmaConsumers, HO::Def, MfmaWawWindow, HC::None,
450
451 // MFMA WAR rules
452 S.addRule()
453 .enabledIf(isMFMAWARRuleEnabled)
454 .producer({HC::MFMA}, HO::Src2, isXDLMFMA)
455 .rawCredit(mfmaReaderRawWindow)
456 .consumer(MfmaConsumers, HO::Def, MfmaWarWindow, HC::None,
458
459 // WMMA WAR.
460 S.addRule()
461 .enabledIf(isWMMAWARABRuleEnabled)
462 .producer({HC::WMMA}, HO::Src0Src1, isXDLWMMA)
463 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
465 .consumer({HC::VMEM | HC::DS}, HO::Def, WmmaCoexecWindow, CoexecCounters,
467
468 // SWMMAC WAR.
469 S.addRule()
470 .enabledIf(isSWMMACWARIndexRuleEnabled)
471 .producer({HC::WMMA}, HO::Src2, isXDLSWMMAC)
472 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
474
475 // WMMA WAW.
476 S.addRule()
477 .enabledIf(isWMMAWAWRuleEnabled)
478 .producer({HC::WMMA}, HO::Def, isXDLWMMA)
479 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
481
482 // TRANS WAR.
483 S.addRule()
484 .enabledIf(isTransWARRuleEnabled)
485 .producer({HC::TRANS}, HO::AnyUse)
486 .consumer({HC::VALU | HC::WMMA}, HO::Def, {/*WindowLength=*/1},
487 CoexecCounters, AntiHintDirection::OneDirectional);
488
489 // Address WAR.
490 S.addRule()
491 .enabledIf(isAddrWARRuleEnabled)
492 .producer({HC::VMEM}, HO::AnyUse)
493 .consumer({HC::WritesVGPR}, HO::Def, {0, addrAntiHintWindow}, HC::VMEM,
495
496 // VA_VDST WAR.
497 S.addRule()
498 .enabledIf(isVAVdstWARRuleEnabled)
499 .lifetime(Lifetime::RegisterCount)
500 .producer({HC::RawVALU}, HO::AnyUse)
501 .consumer({HC::Load}, HO::Def, {0, vaVdstLookbackWindow}, HC::None,
502 AntiHintDirection::Symmetric, isDSorVMEMLoad);
503
504 return S.buildRules();
505}
506
507ArrayRef<HazardAntiHintRule> getAntiHintsRules() {
508 static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules();
509 return Rules;
510}
511
512struct AntiHintWindow {
514 const MachineInstr *Producer = nullptr;
515 unsigned Len = 0;
516 unsigned Elapsed = 0;
517};
518
519using ConsumerTracking = SmallVector<AntiHintWindow, 3>;
520// One per consumer target of a rule.
521using RuleTracking = SmallVector<ConsumerTracking, 3>;
522
523class AntiHintEngine {
524 const HazardContext &Ctx;
526
527 SmallVector<bool, 8> RuleApplies;
528 bool AnyEnabled = false;
529
530public:
531 AntiHintEngine(const HazardContext &Ctx)
532 : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) {
533 for (unsigned R = 0; R < Rules.size(); ++R) {
534 const HazardAntiHintRule &Rule = Rules[R];
535 RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx);
536 AnyEnabled |= RuleApplies[R];
537 }
538 }
539
540 void run(MachineFunction &MF) {
541 if (!AnyEnabled)
542 return;
543
544 SmallVector<RuleTracking, 8> Tracking(Rules.size());
545 for (unsigned R = 0; R < Rules.size(); ++R)
546 Tracking[R].resize(Rules[R].Consumers.size());
547
548 for (const MachineBasicBlock &MBB : MF) {
549 for (RuleTracking &RT : Tracking) {
550 for (ConsumerTracking &Track : RT)
551 Track.clear();
552 }
553 for (const MachineInstr &MI : MBB) {
554 if (MI.isMetaInstruction())
555 continue;
556 const HazardClassMask C = getInstHazardClass(MI, Ctx);
557 // Wait states this instruction contributes to an open window.
558 unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI);
559 addAntiHintsAndExpire(MI, C, WaitStates, Tracking);
560 addWindows(MI, C, Tracking);
561 }
562 }
563 }
564
565private:
566 // Check if the right class and predicate matches.
567 bool sideMatches(const HazardSide &Side, const MachineInstr &MI,
569 return Side.Match.matches(C) &&
570 (!Side.Predicate || Side.Predicate(MI, Ctx));
571 }
572
573 // Producer phase: open a window for each consumer target.
574 void addWindows(const MachineInstr &MI, HazardClassMask C,
576 for (unsigned R = 0; R < Rules.size(); ++R) {
577 const HazardAntiHintRule &Rule = Rules[R];
578 if (!RuleApplies[R] || !sideMatches(Rule.Producer, MI, C))
579 continue;
580
582 // Collect the producer regs.
583 collectOperandRegs(MI, Rule.Producer.Op, Ctx, Regs);
584 if (Regs.empty())
585 continue;
586 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
587 ConsumerIdx != E; ++ConsumerIdx)
588 seedWindow(Rule, Rule.Consumers[ConsumerIdx], MI, Regs,
589 Tracking[R][ConsumerIdx]);
590 }
591 }
592
593 void seedWindow(const HazardAntiHintRule &Rule, const ConsumerTarget &CT,
595 ConsumerTracking &Track) {
596 unsigned Window = resolveWindow(CT, MI, Ctx);
597 if (!Window)
598 return;
599
600 if (Rule.Life == Lifetime::RegisterCount) {
601 // Window length caps the register count.
602 if (Track.empty())
603 Track.push_back({{}, &MI, Window, 0});
604 SmallVectorImpl<Register> &Recent = Track.front().Regs;
605 for (Register Reg : Regs) {
606 if (llvm::is_contained(Recent, Reg))
607 continue;
608 Recent.push_back(Reg);
609 if (Recent.size() > Window)
610 Recent.erase(Recent.begin());
611 }
612 return;
613 }
614
615 Track.push_back({SmallVector<Register, 4>(Regs), &MI, Window, 0});
616 }
617
618 void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
619 const HazardAntiHintRule &Rule,
620 const ConsumerTarget &CT, ConsumerTracking &Track) {
621
622 if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(C))
623 return;
624 for (AntiHintWindow &Window : Track) {
625 // Determine if def generated by producer is read by the MI consumer.
626 bool ReadsProducerDef = llvm::any_of(
627 Window.Producer->all_defs(), [&](const MachineOperand &MO) {
628 return MO.getReg().isVirtual() &&
629 MI.readsVirtualRegister(MO.getReg());
630 });
631 // Advance by RAW window from the producer if that is larger than current
632 // Window.Elapsed.
633 if (ReadsProducerDef) {
634 Window.Elapsed = std::max(
635 Window.Elapsed, Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx));
636 }
637 }
638 }
639
640 // Consumer phase: add anti-hints, then charge and expire open windows.
641 void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C,
642 unsigned WaitStates,
644 for (unsigned R = 0; R < Rules.size(); ++R) {
645 const HazardAntiHintRule &Rule = Rules[R];
646 if (!RuleApplies[R])
647 continue;
648 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
649 ConsumerIdx != E; ++ConsumerIdx) {
650 const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx];
651 ConsumerTracking &Track = Tracking[R][ConsumerIdx];
652 if (Track.empty())
653 continue;
654
655 // Before adding the anti-hints, see if advancing by RAW window will
656 // help remove the window.
657 const bool Counts = Rule.Life == Lifetime::WindowBudget;
658 if (Counts) {
659 advanceByRawWindow(MI, C, Rule, CT, Track);
660 llvm::erase_if(Track, [](const AntiHintWindow &Window) {
661 return Window.Elapsed >= Window.Len;
662 });
663 }
664
665 // Add anti-hints if the consumer matches the instruction.
666 if (sideMatches(CT.Side, MI, C))
667 addAntiHints(CT, MI, Track);
668
669 // This instruction's own wait states count toward the next one.
670 if (Counts && (!CT.CounterMask || (C & CT.CounterMask))) {
671 for (AntiHintWindow &Window : Track)
672 Window.Elapsed += WaitStates;
673 }
674 }
675 }
676 }
677
678 bool isCopyOf(Register Cand, Register HazardReg) const {
679 const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Cand);
680 return Def && Def->isCopy() && Def->getOperand(1).getReg() == HazardReg;
681 }
682
683 void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI,
684 const ConsumerTracking &Track) {
685 if (Track.empty())
686 return;
687 SmallVector<Register, 4> ConsumerRegs;
688 // Collect the consumer regs.
689 collectOperandRegs(MI, CT.Side.Op, Ctx, ConsumerRegs);
690 if (ConsumerRegs.empty())
691 return;
692 SlotIndex Slot = Ctx.LIS->getInstructionIndex(MI).getRegSlot();
693 auto AntiHint = [&](Register ProducerReg) {
694 if (!Ctx.LIS->hasInterval(ProducerReg))
695 return;
696 const LiveInterval &ProducerLI = Ctx.LIS->getInterval(ProducerReg);
697 // Skip a live producer reg.
698 if (ProducerLI.liveAt(Slot))
699 return;
700 for (Register ConsumerReg : ConsumerRegs) {
701 if (ConsumerReg == ProducerReg || isCopyOf(ConsumerReg, ProducerReg))
702 continue;
703
704 Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
706 Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
708 dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
709 << (CT.Direction == AntiHintDirection::Symmetric ? " <-> "
710 : " <- ")
711 << printReg(ConsumerReg, Ctx.TRI) << " (consumer "
712 << Ctx.TII->getName(MI.getOpcode()) << ")\n");
713 }
714 };
715 for (const AntiHintWindow &Window : Track) {
716 for (Register ProducerReg : Window.Regs)
717 AntiHint(ProducerReg);
718 }
719 }
720};
721
722} // namespace
723
725 AntiHintEngine(Ctx).run(MF);
726}
amdgpu AMDGPU DAG DAG Pattern Instruction Selection
AMDGPU Rewrite AGPR Copy MFMA
MachineBasicBlock & MBB
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static Register isCopyOf(const MachineInstr &MI, Register Reg, const TargetInstrInfo &TII)
isFullCopyOf - If MI is a COPY to or from Reg, return the other register, otherwise return 0.
Loop::LoopBounds::Direction Direction
Definition LoopInfo.cpp:249
Register Reg
#define P(N)
Interface definition for SIInstrInfo.
Interface definition for SIRegisterInfo.
This file contains some templates that are useful if you are working with the STL at all.
#define LLVM_DEBUG(...)
Definition Debug.h:119
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
LiveInterval - This class represents the liveness of a register, or stack slot.
bool liveAt(SlotIndex index) const
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
static bool isDGEMM(unsigned Opcode)
static bool isSALU(const MachineInstr &MI)
static bool isSWMMAC(const MachineInstr &MI)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
static bool isTRANS(const MachineInstr &MI)
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
SlotIndex - An opaque wrapper around machine indexes.
Definition SlotIndexes.h:70
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
iterator erase(const_iterator CI)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
uint32_t HazardClassMask
bool(*)(const HazardContext &) RulePredicate
void applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx)
unsigned(*)(const MachineInstr &Producer, HazardClassMask ReaderClass, const HazardContext &Ctx) AdvanceForRawWindowFn
bool(*)(const MachineInstr &, const HazardContext &) InstPredicate
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
DWARFExpression::Operation Op
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2210
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
bool matches(HazardClassMask Mask) const
SmallVector< ConsumerTarget, 3 > Consumers
AdvanceForRawWindowFn AdvanceForRawWindow
unsigned(* Fn)(const MachineInstr &Producer, HazardClassMask ConsumerClass, const HazardContext &Ctx)