27#define DEBUG_TYPE "amdgpu-anti-hints"
43 else if (
TII.isMFMA(
MI))
49 else if (
TII.isDS(
MI))
51 else if (
TII.isVMEM(
MI))
53 else if (
TII.isSMRD(
MI))
55 else if (
TII.isEXP(
MI))
59 else if (
MI.isCopy() &&
MI.getOperand(0).getReg().isVirtual() &&
60 Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(
MI.getOperand(0).getReg())))
67 if (MO.isReg() && MO.getReg().isVirtual() &&
68 Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO.getReg()))) {
80 return Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO.
getReg()));
92 if (isVirtualVGPR(Ctx, MO))
93 Out.push_back(MO.
getReg());
98 TII.getNamedOperand(
MI, AMDGPU::OpName::src0)) {
99 if (isVirtualVGPR(Ctx, *MO))
100 Out.push_back(MO->
getReg());
105 TII.getNamedOperand(
MI, AMDGPU::OpName::src1)) {
106 if (isVirtualVGPR(Ctx, *MO))
107 Out.push_back(MO->
getReg());
112 TII.getNamedOperand(
MI, AMDGPU::OpName::src2)) {
113 if (isVirtualVGPR(Ctx, *MO))
114 Out.push_back(MO->
getReg());
118 for (AMDGPU::OpName Name : {AMDGPU::OpName::src0, AMDGPU::OpName::src1}) {
120 if (isVirtualVGPR(Ctx, *MO))
121 Out.push_back(MO->
getReg());
127 TII.getNamedOperand(
MI, AMDGPU::OpName::idx)) {
128 if (isVirtualVGPR(Ctx, *MO))
129 Out.push_back(MO->
getReg());
134 TII.getNamedOperand(
MI, AMDGPU::OpName::vaddr)) {
135 if (isVirtualVGPR(Ctx, *MO))
136 Out.push_back(MO->
getReg());
140 for (AMDGPU::OpName Name :
141 {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
143 if (isVirtualVGPR(Ctx, *MO))
144 Out.push_back(MO->
getReg());
150 if (isVirtualVGPR(Ctx, MO))
151 Out.push_back(MO.
getReg());
157enum class MFMAHazardKind { RAW, WAW, WAR };
165 const int NumPasses = Ctx.SchedModel->computeInstrLatency(&
MFMA);
169 auto GFX940NPass = [&]() ->
unsigned {
171 ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
174 auto SMFMANPass = [&]() ->
unsigned {
188 case MFMAHazardKind::RAW:
195 return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
200 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
202 case MFMAHazardKind::WAW:
214 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
216 case MFMAHazardKind::WAR:
235 return mfmaWaitStates(
P, MFMAHazardKind::WAW,
HC::None, Ctx);
239 return mfmaWaitStates(
P, MFMAHazardKind::WAR,
HC::None, Ctx);
243 return Ctx.ST->getCLOpts().addr_anti_hint_window;
247 return Ctx.ST->getCLOpts().va_vdst_lookback_window;
250constexpr unsigned NumWMMAHazardCategories = 7;
251unsigned wmmaHazardCategory(
const MachineInstr &Producer,
254 const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA();
255 switch (Ctx.SchedModel->computeInstrLatency(&Producer)) {
259 return IsSWMMAC ? 2 : 0;
261 return LowestRate ? 4 : (IsSWMMAC ? 3 : 1);
265 return NumWMMAHazardCategories;
273 const unsigned Category = wmmaHazardCategory(Producer, Ctx);
274 if (Category >= NumWMMAHazardCategories)
277 static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
278 static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
279 return (ConsumerClass &
HC::WMMA) ? WMMAWaitStates[Category]
280 : VALUWaitStates[Category];
283unsigned mfmaReaderRawWindow(
const MachineInstr &Producer,
286 return mfmaWaitStates(Producer, MFMAHazardKind::RAW, ReaderClass, Ctx);
290 return Ctx.ST->hasGFX90AInsts();
294 return Ctx.ST->hasWMMACoexecutionHazards();
298 return Ctx.ST->hasTransCoexecutionHazard();
302 return Ctx.ST->hasGFX1250Insts();
326 return hasWMMACoexecHazard(Ctx) &&
339 return Ctx.ST->getCLOpts().anti_hints_for_addr && hasGFX1250Insts(Ctx) &&
344 return Ctx.ST->getCLOpts().anti_hints_for_va_vdst && hasGFX1250Insts(Ctx) &&
349 return Ctx.TII->isXDL(
MI);
353 return Ctx.TII->isXDLWMMA(
MI);
361 return MI.mayLoad() && (Ctx.TII->isDS(
MI) || Ctx.TII->isVMEM(
MI));
383 RuleBuilder(HazardRuleSet &S,
unsigned Idx) : S(S),
Idx(
Idx) {}
392 rule().AdvanceForRawWindow = AdvanceForRawWindow;
401 rule().Consumers.push_back(
411 RuleBuilder &lifetime(
Lifetime Life) {
417 RuleBuilder addRule() {
419 return RuleBuilder(*
this, Rules.
size() - 1);
431 const WindowSpec MfmaWawWindow{0, mfmaWawWindow};
432 const WindowSpec MfmaWarWindow{0, mfmaWarWindow};
437 const WindowSpec WmmaCoexecWindow{0, wmmaCoexecWindow};
445 .enabledIf(isMFMAWAWRuleEnabled)
447 .rawCredit(mfmaReaderRawWindow)
448 .consumer(MfmaConsumers, HO::Def, MfmaWawWindow,
HC::None,
453 .enabledIf(isMFMAWARRuleEnabled)
454 .producer({
HC::MFMA}, HO::Src2, isXDLMFMA)
455 .rawCredit(mfmaReaderRawWindow)
456 .consumer(MfmaConsumers, HO::Def, MfmaWarWindow,
HC::None,
461 .enabledIf(isWMMAWARABRuleEnabled)
462 .producer({
HC::WMMA}, HO::Src0Src1, isXDLWMMA)
463 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
465 .consumer({
HC::VMEM |
HC::DS}, HO::Def, WmmaCoexecWindow, CoexecCounters,
470 .enabledIf(isSWMMACWARIndexRuleEnabled)
471 .producer({
HC::WMMA}, HO::Src2, isXDLSWMMAC)
472 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
477 .enabledIf(isWMMAWAWRuleEnabled)
478 .producer({
HC::WMMA}, HO::Def, isXDLWMMA)
479 .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
484 .enabledIf(isTransWARRuleEnabled)
491 .enabledIf(isAddrWARRuleEnabled)
498 .enabledIf(isVAVdstWARRuleEnabled)
504 return S.buildRules();
512struct AntiHintWindow {
516 unsigned Elapsed = 0;
523class AntiHintEngine {
528 bool AnyEnabled =
false;
532 : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.
size()) {
533 for (
unsigned R = 0; R < Rules.
size(); ++R) {
536 AnyEnabled |= RuleApplies[R];
545 for (
unsigned R = 0; R < Rules.
size(); ++R)
546 Tracking[R].resize(Rules[R].Consumers.
size());
549 for (RuleTracking &RT : Tracking) {
550 for (ConsumerTracking &Track : RT)
554 if (
MI.isMetaInstruction())
559 addAntiHintsAndExpire(
MI,
C, WaitStates, Tracking);
560 addWindows(
MI,
C, Tracking);
576 for (
unsigned R = 0; R < Rules.
size(); ++R) {
578 if (!RuleApplies[R] || !sideMatches(Rule.
Producer,
MI,
C))
586 for (
unsigned ConsumerIdx = 0,
E = Rule.
Consumers.size();
587 ConsumerIdx !=
E; ++ConsumerIdx)
588 seedWindow(Rule, Rule.
Consumers[ConsumerIdx],
MI, Regs,
589 Tracking[R][ConsumerIdx]);
595 ConsumerTracking &Track) {
596 unsigned Window = resolveWindow(CT,
MI, Ctx);
603 Track.push_back({{}, &
MI, Window, 0});
609 if (Recent.
size() > Window)
624 for (AntiHintWindow &Window : Track) {
628 return MO.getReg().isVirtual() &&
629 MI.readsVirtualRegister(MO.getReg());
633 if (ReadsProducerDef) {
634 Window.Elapsed = std::max(
644 for (
unsigned R = 0; R < Rules.
size(); ++R) {
648 for (
unsigned ConsumerIdx = 0,
E = Rule.
Consumers.size();
649 ConsumerIdx !=
E; ++ConsumerIdx) {
651 ConsumerTracking &Track = Tracking[R][ConsumerIdx];
659 advanceByRawWindow(
MI,
C, Rule, CT, Track);
661 return Window.Elapsed >= Window.Len;
666 if (sideMatches(CT.
Side,
MI,
C))
667 addAntiHints(CT,
MI, Track);
671 for (AntiHintWindow &Window : Track)
672 Window.Elapsed += WaitStates;
680 return Def &&
Def->isCopy() &&
Def->getOperand(1).getReg() == HazardReg;
684 const ConsumerTracking &Track) {
689 collectOperandRegs(
MI, CT.
Side.
Op, Ctx, ConsumerRegs);
690 if (ConsumerRegs.
empty())
692 SlotIndex Slot = Ctx.LIS->getInstructionIndex(
MI).getRegSlot();
693 auto AntiHint = [&](
Register ProducerReg) {
694 if (!Ctx.LIS->hasInterval(ProducerReg))
696 const LiveInterval &ProducerLI = Ctx.LIS->getInterval(ProducerReg);
698 if (ProducerLI.
liveAt(Slot))
700 for (
Register ConsumerReg : ConsumerRegs) {
701 if (ConsumerReg == ProducerReg ||
isCopyOf(ConsumerReg, ProducerReg))
704 Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
706 Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
708 dbgs() <<
"anti-hint: keep " <<
printReg(ProducerReg, Ctx.TRI)
711 <<
printReg(ConsumerReg, Ctx.TRI) <<
" (consumer "
712 << Ctx.TII->getName(
MI.getOpcode()) <<
")\n");
715 for (
const AntiHintWindow &Window : Track) {
716 for (
Register ProducerReg : Window.Regs)
717 AntiHint(ProducerReg);
725 AntiHintEngine(Ctx).run(MF);
amdgpu AMDGPU DAG DAG Pattern Instruction Selection
AMDGPU Rewrite AGPR Copy MFMA
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static Register isCopyOf(const MachineInstr &MI, Register Reg, const TargetInstrInfo &TII)
isFullCopyOf - If MI is a COPY to or from Reg, return the other register, otherwise return 0.
Loop::LoopBounds::Direction Direction
Interface definition for SIInstrInfo.
Interface definition for SIRegisterInfo.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
LiveInterval - This class represents the liveness of a register, or stack slot.
bool liveAt(SlotIndex index) const
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static bool isDGEMM(unsigned Opcode)
static bool isSALU(const MachineInstr &MI)
static bool isSWMMAC(const MachineInstr &MI)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
static bool isTRANS(const MachineInstr &MI)
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
SlotIndex - An opaque wrapper around machine indexes.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
iterator erase(const_iterator CI)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool(*)(const HazardContext &) RulePredicate
void applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx)
unsigned(*)(const MachineInstr &Producer, HazardClassMask ReaderClass, const HazardContext &Ctx) AdvanceForRawWindowFn
bool(*)(const MachineInstr &, const HazardContext &) InstPredicate
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
DWARFExpression::Operation Op
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
bool matches(HazardClassMask Mask) const
HazardClassMask CounterMask
AntiHintDirection Direction
SmallVector< ConsumerTarget, 3 > Consumers
AdvanceForRawWindowFn AdvanceForRawWindow
unsigned(* Fn)(const MachineInstr &Producer, HazardClassMask ConsumerClass, const HazardContext &Ctx)