28#define DEBUG_TYPE "gcn-hazard-recognizer"
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
34 "Number of WMMA hazard V_NOPs hoisted from loops");
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
40struct MFMAPaddingRatioParser :
public cl::parser<unsigned> {
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg,
unsigned &
Value) {
45 return O.error(
"'" + Arg +
"' value invalid for uint argument!");
48 return O.error(
"'" + Arg +
"' value must be in the range [0, 100]!");
58 cl::desc(
"Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
71 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
72 ST(MF.getSubtarget<
GCNSubtarget>()), TII(*ST.getInstrInfo()),
73 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
74 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
75 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
79 dbgs() <<
" PreRA hazard recognizer: " << MF.getName() <<
"\n";
91 if (CurrentCoExecStage.has_value()) {
92 unsigned Stage = *CurrentCoExecStage;
94 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
95 dbgs() <<
" CoExec window ended at stage " << Stage <<
":\n";
102 EmittedInstrs.clear();
103 EmittedVALUInstrs.clear();
104 HasPendingWMMACoexecHazard =
false;
109void GCNHazardRecognizer::schedulerReset() {
111 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
113 dbgs() <<
" Scheduler Reset: clearing co-exec window, TRANS="
114 << CyclesUntilTRANS <<
", VALU=" << CyclesUntilVALU <<
"\n";
116 CurrentCoExecStage = std::nullopt;
117 CyclesUntilTRANS = 0;
120 CoExecWindowLog.fill(
'.');
123void GCNHazardRecognizer::dumpCoExecWindow()
const {
124 unsigned W = ActiveCoExecInfo.TotalWindow;
129 dbgs() <<
" Stages: ";
130 for (
unsigned I = 0;
I <
W; ++
I)
131 dbgs() <<
I % 10 <<
' ';
135 dbgs() <<
" Slots: ";
136 for (
unsigned I = 0;
I <
W; ++
I)
137 dbgs() << ActiveCoExecInfo.Pattern[
I] <<
' ';
141 dbgs() <<
" Scheduled: ";
142 for (
unsigned I = 0;
I <
W; ++
I)
143 dbgs() << CoExecWindowLog[
I] <<
' ';
147void GCNHazardRecognizer::schedulerAdvanceCycle() {
149 if (CurrentCoExecStage.has_value()) {
150 unsigned Stage = *CurrentCoExecStage;
153 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
155 CoExecWindowLog[Stage] =
'-';
160 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
163 dbgs() <<
" Scheduler AdvanceCycle:";
164 if (CurrentCoExecStage.has_value()) {
165 unsigned Stage = *CurrentCoExecStage;
166 unsigned Next = Stage + 1;
167 if (
Next >= ActiveCoExecInfo.TotalWindow)
168 dbgs() <<
" stage " << Stage <<
"->expired";
170 dbgs() <<
" stage " << Stage <<
"->" <<
Next;
172 if (CyclesUntilTRANS > 0)
173 dbgs() <<
" TRANS=" << CyclesUntilTRANS <<
"->"
174 << (CyclesUntilTRANS - 1);
175 if (CyclesUntilVALU > 0)
176 dbgs() <<
" VALU=" << CyclesUntilVALU <<
"->" << (CyclesUntilVALU - 1);
182 if (CyclesUntilTRANS > 0)
184 if (CyclesUntilVALU > 0)
188 if (CurrentCoExecStage.has_value()) {
189 unsigned Stage = *CurrentCoExecStage + 1;
190 if (Stage >= ActiveCoExecInfo.TotalWindow) {
193 dbgs() <<
" CoExec window complete:\n";
196 CurrentCoExecStage = std::nullopt;
198 CurrentCoExecStage = Stage;
203bool GCNHazardRecognizer::hasCoExecWindowModel()
const {
209 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
213 if (ST.hasGFX950Insts() &&
220void GCNHazardRecognizer::updateWMMAWindowState(
const MachineInstr &
MI) {
221 if (!hasCoExecWindowModel())
231 if (CurrentCoExecStage.has_value()) {
232 unsigned Stage = *CurrentCoExecStage;
234 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
235 dbgs() <<
" CoExec window interrupted at stage " << Stage <<
":\n";
242 CurrentCoExecStage = 0;
243 CoExecWindowLog.fill(
'.');
245 LLVM_DEBUG(
dbgs() <<
" WMMA window started: " << ActiveCoExecInfo.Pattern
246 <<
" (window=" << ActiveCoExecInfo.TotalWindow <<
")\n"
250void GCNHazardRecognizer::updateTRANSState(
const MachineInstr &
MI) {
251 if (!hasCoExecWindowModel())
265 CyclesUntilTRANS = 2;
269void GCNHazardRecognizer::updateMultiCycleVALUState(
const MachineInstr &
MI) {
270 if (!hasCoExecWindowModel())
281 unsigned RepeatRate = TII.getRepeatRate(
MI);
282 if (RepeatRate > 1) {
286 CyclesUntilVALU = RepeatRate;
288 <<
", CyclesUntilVALU=" << CyclesUntilVALU <<
"\n");
298unsigned GCNHazardRecognizer::checkTRANSHazard(
const MachineInstr &
MI)
const {
299 if (!CyclesUntilTRANS)
304 return CyclesUntilTRANS;
308 TII.getRepeatRate(
MI) > 1)
309 return CyclesUntilTRANS;
315GCNHazardRecognizer::checkMultiCycleVALUHazard(
const MachineInstr &
MI)
const {
316 if (!CyclesUntilVALU)
327 return CyclesUntilVALU;
331GCNHazardRecognizer::checkWMMACoexecSlot(
const MachineInstr &
MI)
const {
333 if (!CurrentCoExecStage.has_value())
336 unsigned Stage = *CurrentCoExecStage;
338 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
341 if (StallCycles == 0)
345 unsigned NextStage = Stage + StallCycles;
346 if (NextStage < ActiveCoExecInfo.TotalWindow) {
349 dbgs() <<
" CoExec stall: stage=" << Stage <<
"("
352 <<
" -> stall " << StallCycles <<
" (next allowed=" << NextStage
361 dbgs() <<
" CoExec stall: stage=" << Stage <<
"("
364 << StallCycles <<
" (window ends)\n"
370GCNHazardRecognizer::checkMultiShadowHazard(
const MachineInstr &
MI)
const {
372 if (!hasCoExecWindowModel())
376 if (!CurrentCoExecStage.has_value())
379 if (!CyclesUntilTRANS)
388 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
390 return CyclesUntilTRANS +
391 ActiveCoExecInfo.getStallCycles(InstMask, LookAheadStage);
394void GCNHazardRecognizer::schedulerEmitInstruction(
MachineInstr *
MI) {
396 bool InWindow = CurrentCoExecStage.has_value();
397 bool HasActiveState =
398 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
399 if (HasActiveState) {
401 unsigned Stage = *CurrentCoExecStage;
402 dbgs() <<
" Stage " << Stage <<
"("
415 bool HasActiveState = CurrentCoExecStage.has_value() ||
416 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
422 updateWMMAWindowState(*
MI);
423 updateTRANSState(*
MI);
424 updateMultiCycleVALUState(*
MI);
434 schedulerEmitInstruction(
MI);
438 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
442 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
447 case AMDGPU::S_SETREG_B32:
448 case AMDGPU::S_SETREG_B32_mode:
449 case AMDGPU::S_SETREG_IMM32_B32:
450 case AMDGPU::S_SETREG_IMM32_B32_mode:
457 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
461 return Opcode == AMDGPU::S_RFE_B64;
466 case AMDGPU::S_MOVRELS_B32:
467 case AMDGPU::S_MOVRELS_B64:
468 case AMDGPU::S_MOVRELD_B32:
469 case AMDGPU::S_MOVRELD_B64:
478 if (
TII.isAlwaysGDS(
MI.getOpcode()))
481 switch (
MI.getOpcode()) {
482 case AMDGPU::S_SENDMSG:
483 case AMDGPU::S_SENDMSGHALT:
484 case AMDGPU::S_TTRACEDATA:
488 case AMDGPU::DS_PERMUTE_B32:
489 case AMDGPU::DS_BPERMUTE_B32:
492 if (
TII.isDS(
MI.getOpcode())) {
493 int GDS = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
494 AMDGPU::OpName::gds);
495 if (
MI.getOperand(GDS).getImm())
503 unsigned Opcode =
MI.getOpcode();
504 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
505 Opcode == AMDGPU::V_PERMLANE64_B32 ||
506 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
507 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
508 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
509 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
510 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
511 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
512 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
513 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
514 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
515 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
517 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
526 AMDGPU::OpName::simm16);
542 if (checkMultiShadowHazard(*
MI) > 0)
544 if (checkWMMACoexecSlot(*
MI) > 0)
546 if (checkTRANSHazard(*
MI) > 0)
548 if (checkMultiCycleVALUHazard(*
MI) > 0)
558 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(
MI) > 0)
561 if (checkFPAtomicToDenormModeHazard(
MI) > 0)
566 if (checkWMMACoexecutionHazards(
MI) > 0) {
567 HasPendingWMMACoexecHazard =
true;
572 if (ST.hasNoDataDepHazard())
579 checkVALUHazards(
MI) > 0)
585 if (
isDivFMas(
MI->getOpcode()) && checkDivFMasHazards(
MI) > 0)
588 if (
isRWLane(
MI->getOpcode()) && checkRWLaneHazards(
MI) > 0)
594 checkMAIVALUHazards(
MI) > 0)
597 if (
isSGetReg(
MI->getOpcode()) && checkGetRegHazards(
MI) > 0)
600 if (
isSSetReg(
MI->getOpcode()) && checkSetRegHazards(
MI) > 0)
603 if (
isRFE(
MI->getOpcode()) && checkRFEHazards(
MI) > 0)
606 if (((ST.hasReadM0MovRelInterpHazard() &&
608 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
609 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
611 (ST.hasReadM0LdsDmaHazard() &&
isLdsDma(*
MI)) ||
612 (ST.hasReadM0LdsDirectHazard() &&
613 MI->readsRegister(AMDGPU::LDS_DIRECT,
nullptr))) &&
614 checkReadM0Hazards(
MI) > 0)
621 checkMAILdStHazards(
MI) > 0)
624 if (ST.hasPermlaneForwardingHazard() &&
isPermlane(*
MI) &&
625 checkPermlaneHazards(
MI) > 0)
628 if (
MI->isInlineAsm() && checkInlineAsmHazards(
MI) > 0)
636 while (Quantity > 0) {
637 unsigned Arg = std::min(Quantity, 8u);
645GCNHazardRecognizer::getMFMAPipelineWaitStates(
const MachineInstr &
MI)
const {
646 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&
MI);
647 assert(TSchedModel.getWriteProcResBegin(SC) !=
648 TSchedModel.getWriteProcResEnd(SC));
649 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
652void GCNHazardRecognizer::processBundle() {
656 for (;
MI !=
E &&
MI->isInsideBundle(); ++
MI) {
657 CurrCycleInstr = &*
MI;
661 fixHazards(CurrCycleInstr);
669 for (
unsigned i = 0, e = std::min(WaitStates,
MaxLookAhead - 1); i <
e; ++i)
670 EmittedInstrs.push_front(
nullptr);
672 EmittedInstrs.push_front(CurrCycleInstr);
675 CurrCycleInstr =
nullptr;
683 if (
MI->isInsideBundle())
697 CurrCycleInstr =
nullptr;
698 return std::max(W, ST.getCLOpts().snop_padding);
706 W = checkWMMACoexecSlot(*
MI);
707 W = std::max(W, checkTRANSHazard(*
MI));
708 W = std::max(W, checkMultiCycleVALUHazard(*
MI));
709 W = std::max(W, checkMultiShadowHazard(*
MI));
725 return std::max(WaitStates, checkSMRDHazards(
MI));
727 if (ST.hasNSAtoVMEMBug())
728 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(
MI));
730 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(
MI));
732 if (ST.hasNoDataDepHazard())
736 WaitStates = std::max(WaitStates, checkVMEMHazards(
MI));
739 WaitStates = std::max(WaitStates, checkVALUHazards(
MI));
742 WaitStates = std::max(WaitStates, checkDPPHazards(
MI));
745 WaitStates = std::max(WaitStates, checkDivFMasHazards(
MI));
748 WaitStates = std::max(WaitStates, checkRWLaneHazards(
MI));
753 WaitStates = std::max(WaitStates, checkMAIVALUHazards(
MI));
755 if (
MI->isInlineAsm())
756 return std::max(WaitStates, checkInlineAsmHazards(
MI));
759 return std::max(WaitStates, checkGetRegHazards(
MI));
762 return std::max(WaitStates, checkSetRegHazards(
MI));
765 return std::max(WaitStates, checkRFEHazards(
MI));
767 if ((ST.hasReadM0MovRelInterpHazard() &&
769 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
770 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
772 (ST.hasReadM0LdsDmaHazard() &&
isLdsDma(*
MI)) ||
773 (ST.hasReadM0LdsDirectHazard() &&
774 MI->readsRegister(AMDGPU::LDS_DIRECT,
nullptr)))
775 return std::max(WaitStates, checkReadM0Hazards(
MI));
778 return std::max(WaitStates, checkMAIHazards(
MI));
781 return std::max(WaitStates, checkMAILdStHazards(
MI));
783 if (ST.hasPermlaneForwardingHazard() &&
isPermlane(*
MI))
784 return std::max(WaitStates, checkPermlaneHazards(
MI));
790 EmittedInstrs.push_front(
nullptr);
795 schedulerAdvanceCycle();
799 if (!CurrCycleInstr) {
800 EmittedInstrs.push_front(
nullptr);
802 if (HasPendingWMMACoexecHazard)
803 EmittedVALUInstrs.push_front(
nullptr);
807 HasPendingWMMACoexecHazard =
false;
809 if (CurrCycleInstr->isBundle()) {
814 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
815 if (!NumWaitStates) {
816 CurrCycleInstr =
nullptr;
821 EmittedInstrs.push_front(CurrCycleInstr);
828 EmittedVALUInstrs.push_front(CurrCycleInstr);
834 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() ==
nullptr)
835 EmittedVALUInstrs.pop_front();
843 EmittedInstrs.push_front(
nullptr);
850 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
851 EmittedVALUInstrs.resize(MaxVALULookAhead);
853 CurrCycleInstr =
nullptr;
858 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
868template <
typename StateT>
878 static bool isEqual(
const StateMapKey &
LHS,
const StateMapKey &
RHS) {
883 static unsigned getHashValue(
const StateMapKey &
Key) {
884 return StateT::getHashValue((*
Key.States)[
Key.Idx]);
886 static unsigned getHashValue(
const StateT &State) {
887 return StateT::getHashValue(State);
889 static bool isEqual(
const StateMapKey &
LHS,
const StateMapKey &
RHS) {
890 return StateT::isEqual((*
LHS.States)[
LHS.Idx], (*
RHS.States)[
RHS.Idx]);
892 static bool isEqual(
const StateT &
LHS,
const StateMapKey &
RHS) {
893 return StateT::isEqual(
LHS, (*
RHS.States)[
RHS.Idx]);
902 StateT State = InitialState;
905 unsigned WorkIdx = 0;
907 bool Expired =
false;
908 for (
auto E =
MBB->instr_rend();
I !=
E; ++
I) {
913 auto Result = IsHazard(State, *
I);
921 if (
I->isInlineAsm() ||
I->isMetaInstruction())
924 UpdateState(State, *
I);
928 size_t StateIdx = States.
size();
929 StateMapKey
Key = {&States, StateIdx};
930 auto Insertion = StateMap.
insert_as(std::pair(
Key, StateIdx), State);
931 if (Insertion.second) {
934 StateIdx = Insertion.first->second;
940 if (WorkIdx == Worklist.
size())
944 std::tie(
MBB, StateIdx) = Worklist[WorkIdx++];
945 State = States[StateIdx];
946 I =
MBB->instr_rbegin();
959 unsigned Unexecuted = 0, Passed = 0;
960 for (
auto I =
Pred.instr_rbegin(),
E =
Pred.instr_rend();
961 I !=
E &&
I->isTerminator(); ++
I, ++Passed) {
964 if (
I->isBranch() &&
I->getNumOperands() > 0 &&
I->getOperand(0).isMBB() &&
965 TII.getBranchDestBlock(*
I) ==
MBB)
976 return MI.isInlineAsm() ? 0 : GetNumWaitStates(
MI);
985class WindowDeficitSearch {
988 function_ref<std::optional<int>(
const MachineInstr &)> WindowFor,
992 : WindowFor(WindowFor), GetNumWaitStates(GetNumWaitStates),
993 MaxWindow(MaxWindow) {}
995 int run(
const MachineInstr &
MI);
1002 const MachineBasicBlock *
MBB;
1007 void arrive(
const MachineBasicBlock *
MBB,
1010 void scan(Arrival
A);
1012 const function_ref<std::optional<int>(
const MachineInstr &)> WindowFor;
1014 const int MaxWindow;
1021 DenseMap<const MachineBasicBlock *, int> Best;
1026void WindowDeficitSearch::arrive(
1031 if (MaxWindow - Distance <= Deficit)
1034 if (Pending.empty())
1035 Pending.resize(MaxWindow);
1037 auto [It,
Inserted] = Best.try_emplace(
MBB, Distance);
1039 if (It->second <= Distance)
1041 It->second = Distance;
1044 assert(Distance < MaxWindow &&
"arrival outside the window");
1045 Pending[Distance].push_back({
MBB,
Start, Distance});
1050void WindowDeficitSearch::scan(Arrival
A) {
1052 if (Best.lookup(
A.MBB) <
A.Distance)
1055 int Distance =
A.Distance;
1056 for (
auto I =
A.Start,
E =
A.MBB->instr_rend();
I !=
E; ++
I) {
1060 if (std::optional<int> Window = WindowFor(*
I)) {
1061 assert(*Window >= 0 && *Window <= MaxWindow &&
"window out of bounds");
1062 Deficit = std::max(Deficit, *Window - Distance);
1067 if (Distance >= MaxWindow)
1078 assert(!WindowFor(*Start) &&
"terminators must not ask for a window");
1082 arrive(
Pred, Start, Distance + Executed);
1089 scan({
MI.getParent(), std::next(
MI.getReverseIterator()), 0});
1095 for (
int Distance = 0; Distance < static_cast<int>(Pending.size());
1097 for (
unsigned I = 0;
I != Pending[Distance].size(); ++
I)
1098 scan(Pending[Distance][
I]);
1114 for (
auto E =
MBB->instr_rend();
I !=
E; ++
I) {
1119 if (
I->isMetaInstruction())
1125 if (
I->isInlineAsm())
1128 WaitStates += GetNumWaitStates(*
I);
1130 if (IsExpired(*
I, WaitStates))
1131 return std::numeric_limits<int>::max();
1134 int MinWaitStates = std::numeric_limits<int>::max();
1140 IsExpired, Visited, GetNumWaitStates);
1142 MinWaitStates = std::min(MinWaitStates, W);
1145 return MinWaitStates;
1156 std::next(
MI->getReverseIterator()), 0, IsExpired,
1157 Visited, GetNumWaitStates);
1160int GCNHazardRecognizer::getMaxWindowDeficit(
int MaxWindow,
1161 WindowForFn WindowFor)
const {
1163 return WindowDeficitSearch(WindowFor, MaxWindow).run(*CurrCycleInstr);
1167 int Deficit = 0, Distance = 0;
1168 for (MachineInstr *
MI : EmittedInstrs) {
1170 if (std::optional<int> Window = WindowFor(*
MI)) {
1171 assert(*Window >= 0 && *Window <= MaxWindow &&
"window out of bounds");
1172 Deficit = std::max(Deficit, *Window - Distance);
1179 if (Distance >= MaxWindow)
1185int GCNHazardRecognizer::getWaitStatesSince(
1186 IsHazardFn IsHazard,
int Limit, GetNumWaitStatesFn GetNumWaitStates)
const {
1188 auto IsExpiredFn = [Limit](
const MachineInstr &,
int WaitStates) {
1189 return WaitStates >= Limit;
1191 return ::getWaitStatesSince(IsHazard, CurrCycleInstr,
IsExpiredFn,
1196 for (MachineInstr *
MI : EmittedInstrs) {
1201 if (
MI->isInlineAsm())
1204 WaitStates +=
MI ? GetNumWaitStates(*
MI) : 1;
1206 if (WaitStates >= Limit)
1209 return std::numeric_limits<int>::max();
1212int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1217int GCNHazardRecognizer::getMaxVALUWindowDeficit(
int MaxWindow,
1218 WindowForFn WindowFor)
const {
1220 auto GetVALUWaitStates = [](
const MachineInstr &
MI) ->
unsigned {
1223 return WindowDeficitSearch(WindowFor, MaxWindow, GetVALUWaitStates)
1224 .run(*CurrCycleInstr);
1230 assert(MaxWindow <=
static_cast<int>(MaxVALULookAhead) &&
1231 "MaxWindow exceeds the EmittedVALUInstrs lookahead window");
1233 for (MachineInstr *
MI : EmittedVALUInstrs) {
1235 if (std::optional<int> Window = WindowFor(*
MI)) {
1236 assert(*Window >= 0 && *Window <= MaxWindow &&
"window out of bounds");
1237 return *Window - WaitStates;
1243 if (WaitStates >= MaxWindow)
1249int GCNHazardRecognizer::getWaitStatesSinceDef(
unsigned Reg,
1250 IsHazardFn IsHazardDef,
1252 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1255 return IsHazardDef(
MI) &&
MI.modifiesRegister(
Reg, TRI);
1258 return getWaitStatesSince(
IsHazardFn, Limit);
1261int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1267 return getWaitStatesSince(
IsHazardFn, Limit);
1276 for (MCRegUnit Unit :
TRI.regunits(
Reg))
1277 BV.
set(
static_cast<unsigned>(Unit));
1289void GCNHazardRecognizer::addClauseInst(
const MachineInstr &
MI)
const {
1301int GCNHazardRecognizer::checkSoftClauseHazards(
MachineInstr *MEM)
const {
1304 if (!ST.isXNACKEnabled())
1307 bool IsSMRD = TII.isSMRD(*MEM);
1321 for (MachineInstr *
MI : EmittedInstrs) {
1333 if (ClauseDefs.none())
1339 if (
MEM->mayStore())
1342 addClauseInst(*MEM);
1346 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
1349int GCNHazardRecognizer::checkSMRDHazards(
MachineInstr *SMRD)
const {
1350 int WaitStatesNeeded = 0;
1352 WaitStatesNeeded = checkSoftClauseHazards(SMRD);
1355 if (!ST.hasSMRDReadVALUDefHazard())
1356 return WaitStatesNeeded;
1360 int SmrdSgprWaitStates = 4;
1361 auto IsHazardDefFn = [
this](
const MachineInstr &
MI) {
1362 return TII.isVALU(
MI,
false);
1364 auto IsBufferHazardDefFn = [
this](
const MachineInstr &
MI) {
1365 return TII.isSALU(
MI);
1368 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
1370 for (
const MachineOperand &Use :
SMRD->uses()) {
1373 int WaitStatesNeededForUse =
1374 SmrdSgprWaitStates - getWaitStatesSinceDef(
Use.getReg(), IsHazardDefFn,
1375 SmrdSgprWaitStates);
1376 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1386 int WaitStatesNeededForUse =
1387 SmrdSgprWaitStates - getWaitStatesSinceDef(
Use.getReg(),
1388 IsBufferHazardDefFn,
1389 SmrdSgprWaitStates);
1390 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1394 return WaitStatesNeeded;
1397int GCNHazardRecognizer::checkVMEMHazards(
MachineInstr *VMEM)
const {
1398 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1401 int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
1405 const int VmemSgprWaitStates = 5;
1406 auto IsHazardDefFn = [
this](
const MachineInstr &
MI) {
1407 return TII.isVALU(
MI,
false);
1409 for (
const MachineOperand &Use :
VMEM->uses()) {
1410 if (!
Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(),
Use.getReg()))
1413 int WaitStatesNeededForUse =
1414 VmemSgprWaitStates - getWaitStatesSinceDef(
Use.getReg(), IsHazardDefFn,
1415 VmemSgprWaitStates);
1416 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1418 return WaitStatesNeeded;
1421int GCNHazardRecognizer::checkDPPHazards(
MachineInstr *DPP)
const {
1422 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1423 const SIInstrInfo *TII = ST.getInstrInfo();
1426 int DppVgprWaitStates = 2;
1427 int DppExecWaitStates = 5;
1428 int WaitStatesNeeded = 0;
1429 auto IsHazardDefFn = [TII](
const MachineInstr &
MI) {
1430 return TII->isVALU(
MI,
false);
1433 for (
const MachineOperand &Use :
DPP->uses()) {
1434 if (!
Use.isReg() || !TRI->isVGPR(MF.getRegInfo(),
Use.getReg()))
1436 int WaitStatesNeededForUse =
1437 DppVgprWaitStates - getWaitStatesSinceDef(
1439 [](
const MachineInstr &) { return true; },
1441 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1444 WaitStatesNeeded = std::max(
1446 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
1447 DppExecWaitStates));
1449 return WaitStatesNeeded;
1452int GCNHazardRecognizer::checkDivFMasHazards(
MachineInstr *DivFMas)
const {
1453 const SIInstrInfo *TII = ST.getInstrInfo();
1457 const int DivFMasWaitStates = 4;
1458 auto IsHazardDefFn = [TII](
const MachineInstr &
MI) {
1459 return TII->isVALU(
MI,
false);
1461 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
1464 return DivFMasWaitStates - WaitStatesNeeded;
1467int GCNHazardRecognizer::checkGetRegHazards(
MachineInstr *GetRegInstr)
const {
1468 const SIInstrInfo *TII = ST.getInstrInfo();
1469 unsigned GetRegHWReg =
getHWReg(TII, *GetRegInstr);
1471 const int GetRegWaitStates = 2;
1472 auto IsHazardFn = [TII, GetRegHWReg](
const MachineInstr &
MI) {
1475 int WaitStatesNeeded = getWaitStatesSinceSetReg(
IsHazardFn, GetRegWaitStates);
1477 return GetRegWaitStates - WaitStatesNeeded;
1480int GCNHazardRecognizer::checkSetRegHazards(
MachineInstr *SetRegInstr)
const {
1481 const SIInstrInfo *TII = ST.getInstrInfo();
1482 unsigned HWReg =
getHWReg(TII, *SetRegInstr);
1484 const int SetRegWaitStates = ST.getSetRegWaitStates();
1485 auto IsHazardFn = [TII, HWReg](
const MachineInstr &
MI) {
1488 int WaitStatesNeeded = getWaitStatesSinceSetReg(
IsHazardFn, SetRegWaitStates);
1489 return SetRegWaitStates - WaitStatesNeeded;
1492int GCNHazardRecognizer::createsVALUHazard(
const MachineInstr &
MI)
const {
1496 const SIInstrInfo *TII = ST.getInstrInfo();
1497 unsigned Opcode =
MI.getOpcode();
1498 const MCInstrDesc &
Desc =
MI.getDesc();
1500 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
1503 VDataRCID = TII->getOpRegClassID(
Desc.operands()[VDataIdx]);
1505 if (TII->isMUBUF(
MI) || TII->isMTBUF(
MI)) {
1515 if (ST.hasVDecCoExecHazard())
1517 const MachineOperand *SOffset =
1518 TII->getNamedOperand(
MI, AMDGPU::OpName::soffset);
1519 if (!SOffset || !SOffset->
isReg())
1528 if (TII->isMIMG(
MI)) {
1529 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
1531 Desc.operands()[SRsrcIdx])) == 256);
1535 if (TII->isFLAT(
MI)) {
1547int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1552 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1554 auto IsHazard = [&](
const MachineInstr &
MI) {
1555 int DataIdx = createsVALUHazard(
MI);
1556 return DataIdx >= 0 &&
1557 TRI->regsOverlap(
MI.getOperand(DataIdx).getReg(),
Reg);
1560 return std::max(0, 1 - getWaitStatesSince(IsHazard, 1));
1563int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1570 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1571 const SIInstrInfo *TII = ST.getInstrInfo();
1573 int WaitStatesNeeded = 0;
1577 for (
int Window = 1; Window <= 2; ++Window) {
1578 auto IsHazard = [&](
const MachineInstr &
MI) {
1579 int DataIdx = createsVALUHazard(
MI);
1581 !TRI->regsOverlap(
MI.getOperand(DataIdx).getReg(),
Reg))
1586 if (Window == 1 || !TII->isBUF(
MI))
1589 const MachineOperand *SOffset =
1590 TII->getNamedOperand(
MI, AMDGPU::OpName::soffset);
1591 return !SOffset || !SOffset->
isReg();
1593 WaitStatesNeeded = std::max(WaitStatesNeeded,
1594 Window - getWaitStatesSince(IsHazard, Window));
1597 return WaitStatesNeeded;
1600int GCNHazardRecognizer::checkVALUHazardsHelper(
1605 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1607 if (!TRI->isVectorRegister(
MRI,
Def.getReg()))
1610 if (ST.hasVDecCoExecHazard())
1611 return checkSOFFSETWindowVALUHazardsHelper(
Def.getReg());
1613 return checkUniformWindowVALUHazardsHelper(
Def.getReg());
1629 unsigned Opcode =
MI.getOpcode();
1639 if (
auto *DstSel =
TII->getNamedOperand(
MI, AMDGPU::OpName::dst_sel))
1641 return TII->getNamedOperand(
MI, AMDGPU::OpName::vdst);
1649 (
TII->getNamedImmOperand(
MI, AMDGPU::OpName::src0_modifiers) &
1651 return TII->getNamedOperand(
MI, AMDGPU::OpName::vdst);
1659 return TII->getNamedOperand(
MI, AMDGPU::OpName::vdst);
1663 (
TII->getNamedImmOperand(
MI, AMDGPU::OpName::src2_modifiers) &
1665 return TII->getNamedOperand(
MI, AMDGPU::OpName::vdst);
1671 return TII->getNamedOperand(
MI, AMDGPU::OpName::vdst);
1692 for (
auto &Operand : VALU->operands()) {
1693 if (Operand.isReg() &&
TRI->regsOverlap(Dst->getReg(), Operand.getReg())) {
1700int GCNHazardRecognizer::checkVALUHazards(
MachineInstr *VALU)
const {
1701 int WaitStatesNeeded = 0;
1704 const int TransDefWaitstates = 1;
1706 auto IsTransDefFn = [
this,
VALU](
const MachineInstr &
MI) {
1709 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1710 const SIInstrInfo *TII = ST.getInstrInfo();
1711 Register Def = TII->getNamedOperand(
MI, AMDGPU::OpName::vdst)->getReg();
1713 for (
const MachineOperand &Use :
VALU->explicit_uses()) {
1714 if (
Use.isReg() && TRI->regsOverlap(Def,
Use.getReg()))
1721 int WaitStatesNeededForDef =
1722 TransDefWaitstates -
1723 getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
1724 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1727 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1728 const int Shift16DefWaitstates = 1;
1730 auto IsShift16BitDefFn = [
this,
VALU](
const MachineInstr &ProducerMI) {
1731 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1732 const MachineOperand *ForwardedDst =
1738 if (ProducerMI.isInlineAsm()) {
1740 for (
auto &Def : ProducerMI.all_defs()) {
1749 int WaitStatesNeededForDef =
1750 Shift16DefWaitstates -
1751 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
1752 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1755 if (ST.hasVDecCoExecHazard()) {
1756 const int VALUWriteSGPRVALUReadWaitstates = 2;
1757 const int VALUWriteEXECRWLane = 4;
1758 const int VALUWriteVGPRReadlaneRead = 1;
1760 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1761 const MachineRegisterInfo &
MRI = MF.getRegInfo();
1763 auto IsVALUDefSGPRFn = [&
UseReg, TRI](
const MachineInstr &
MI) {
1766 return MI.modifiesRegister(
UseReg, TRI);
1769 for (
const MachineOperand &Use :
VALU->explicit_uses()) {
1775 int WaitStatesNeededForDef =
1776 VALUWriteSGPRVALUReadWaitstates -
1777 getWaitStatesSince(IsVALUDefSGPRFn,
1778 VALUWriteSGPRVALUReadWaitstates);
1779 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1783 if (
VALU->readsRegister(AMDGPU::VCC, TRI)) {
1785 int WaitStatesNeededForDef =
1786 VALUWriteSGPRVALUReadWaitstates -
1787 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
1788 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1791 switch (
VALU->getOpcode()) {
1792 case AMDGPU::V_READLANE_B32:
1793 case AMDGPU::V_READFIRSTLANE_B32: {
1794 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
1796 int WaitStatesNeededForDef =
1797 VALUWriteVGPRReadlaneRead -
1798 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
1799 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1802 case AMDGPU::V_WRITELANE_B32: {
1804 int WaitStatesNeededForDef =
1805 VALUWriteEXECRWLane -
1806 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
1807 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1817 if (!ST.has12DWordStoreHazard())
1818 return WaitStatesNeeded;
1820 const MachineRegisterInfo &
MRI = MF.getRegInfo();
1822 for (
const MachineOperand &Def :
VALU->defs()) {
1823 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def,
MRI));
1826 return WaitStatesNeeded;
1829int GCNHazardRecognizer::checkInlineAsmHazards(
MachineInstr *IA)
const {
1838 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1839 !ST.hasCvtScaleForwardingHazard())
1842 const MachineRegisterInfo &
MRI = MF.getRegInfo();
1843 int WaitStatesNeeded = 0;
1845 for (
const MachineOperand &
Op :
1847 if (
Op.isReg() &&
Op.isDef()) {
1848 if (!TRI.isVectorRegister(
MRI,
Op.getReg()))
1851 if (ST.has12DWordStoreHazard()) {
1853 std::max(WaitStatesNeeded, checkVALUHazardsHelper(
Op,
MRI));
1858 if (ST.hasDstSelForwardingHazard()) {
1859 const int Shift16DefWaitstates = 1;
1861 auto IsShift16BitDefFn = [
this, &
IA](
const MachineInstr &ProducerMI) {
1865 return IA->modifiesRegister(Dst->getReg(), &TRI) ||
1866 IA->readsRegister(Dst->getReg(), &TRI);
1868 if (ProducerMI.isInlineAsm()) {
1870 for (
auto &Def : ProducerMI.all_defs()) {
1871 if (
IA->modifiesRegister(
Def.getReg(), &TRI) ||
1872 IA->readsRegister(
Def.getReg(), &TRI)) {
1881 int WaitStatesNeededForDef =
1882 Shift16DefWaitstates -
1883 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
1884 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1887 return WaitStatesNeeded;
1890int GCNHazardRecognizer::checkRWLaneHazards(
MachineInstr *RWLane)
const {
1891 const SIInstrInfo *TII = ST.getInstrInfo();
1892 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1893 const MachineRegisterInfo &
MRI = MF.getRegInfo();
1895 const MachineOperand *LaneSelectOp =
1896 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
1898 if (!LaneSelectOp->
isReg() || !TRI->isSGPRReg(
MRI, LaneSelectOp->
getReg()))
1903 return TII->isVALU(
MI,
false);
1906 const int RWLaneWaitStates = 4;
1907 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg,
IsHazardFn,
1909 return RWLaneWaitStates - WaitStatesSince;
1912int GCNHazardRecognizer::checkRFEHazards(
MachineInstr *RFE)
const {
1913 if (!ST.hasRFEHazards())
1916 const SIInstrInfo *TII = ST.getInstrInfo();
1918 const int RFEWaitStates = 1;
1923 int WaitStatesNeeded = getWaitStatesSinceSetReg(
IsHazardFn, RFEWaitStates);
1924 return RFEWaitStates - WaitStatesNeeded;
1927int GCNHazardRecognizer::checkReadM0Hazards(
MachineInstr *
MI)
const {
1928 const SIInstrInfo *TII = ST.getInstrInfo();
1929 const int ReadM0WaitStates = 1;
1930 auto IsHazardFn = [TII](
const MachineInstr &
MI) {
return TII->isSALU(
MI); };
1931 return ReadM0WaitStates -
1932 getWaitStatesSinceDef(AMDGPU::M0,
IsHazardFn, ReadM0WaitStates);
1937 int WaitStatesNeeded,
bool IsHoisting) {
1939 for (
int I = 0;
I < WaitStatesNeeded; ++
I)
1940 BuildMI(
MBB, InsertPt,
DL, TII.get(AMDGPU::V_NOP_e32));
1944 fixVMEMtoScalarWriteHazards(
MI);
1945 fixVcmpxPermlaneHazards(
MI);
1946 fixSMEMtoVectorWriteHazards(
MI);
1947 fixVcmpxExecWARHazard(
MI);
1948 fixLdsBranchVmemWARHazard(
MI);
1949 if (ST.hasLdsDirect()) {
1950 fixLdsDirectVALUHazard(
MI);
1951 fixLdsDirectVMEMHazard(
MI);
1953 fixVALUPartialForwardingHazard(
MI);
1954 fixVALUTransUseHazard(
MI);
1955 fixVALUTransCoexecutionHazards(
MI);
1957 fixWMMACoexecutionHazards(
MI);
1958 fixShift64HighRegBug(
MI);
1959 fixVALUMaskWriteHazard(
MI);
1960 fixRequiredExportPriority(
MI);
1961 if (ST.hasVPermPk16Hazard())
1962 fixVPermPk16Hazard(
MI);
1963 if (ST.requiresWaitIdleBeforeGetReg())
1964 fixGetRegWaitIdle(
MI);
1965 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1966 fixDsAtomicAsyncBarrierArriveB64(
MI);
1967 if (ST.hasScratchBaseForwardingHazard())
1968 fixScratchBaseForwardingHazard(
MI);
1969 if (ST.setRegModeNeedsVNOPs())
1971 if (ST.hasNeedsTDMDrain())
1973 if (ST.hasTransF16ToPartialVGPRWriteHazard())
1974 fixTransF16ToPartialVGPRWriteHazard(
MI);
1979 return (
TII.isVOPC(
MI) ||
1980 (
MI.isCompare() && (
TII.isVOP3(
MI) ||
TII.isSDWA(
MI)))) &&
1981 MI.modifiesRegister(AMDGPU::EXEC, &
TRI);
1984bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(
MachineInstr *
MI) {
1988 const SIInstrInfo *TII = ST.getInstrInfo();
1989 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1995 unsigned Opc =
MI.getOpcode();
1997 Opc != AMDGPU::V_NOP_e32 &&
Opc != AMDGPU::V_NOP_e64 &&
1998 Opc != AMDGPU::V_NOP_sdwa;
2002 std::numeric_limits<int>::max())
2008 auto *
Src0 = TII->getNamedOperand(*
MI, AMDGPU::OpName::src0);
2010 bool IsUndef =
Src0->isUndef();
2012 TII->get(AMDGPU::V_MOV_B32_e32))
2019bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(
MachineInstr *
MI) {
2020 if (!ST.hasVMEMtoScalarWriteHazard())
2022 assert(!ST.hasExtendedWaitCounts());
2027 if (
MI->getNumDefs() == 0)
2030 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2036 for (
const MachineOperand &Def :
MI->defs()) {
2037 const MachineOperand *
Op =
2038 I.findRegisterUseOperand(
Def.getReg(), TRI,
false);
2048 (
MI.getOpcode() == AMDGPU::S_WAITCNT &&
2049 !
MI.getOperand(0).getImm()) ||
2050 (
MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2052 static_cast<unsigned>(
MI.getOperand(0).getImm())) == 0);
2056 std::numeric_limits<int>::max())
2059 const SIInstrInfo *TII = ST.getInstrInfo();
2061 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
2066bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(
MachineInstr *
MI) {
2067 if (!ST.hasSMEMtoVectorWriteHazard())
2069 assert(!ST.hasExtendedWaitCounts());
2074 AMDGPU::OpName SDSTName;
2075 switch (
MI->getOpcode()) {
2076 case AMDGPU::V_READLANE_B32:
2077 case AMDGPU::V_READFIRSTLANE_B32:
2078 SDSTName = AMDGPU::OpName::vdst;
2081 SDSTName = AMDGPU::OpName::sdst;
2085 const SIInstrInfo *TII = ST.getInstrInfo();
2086 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2088 const MachineOperand *SDST = TII->getNamedOperand(*
MI, SDSTName);
2090 for (
const auto &MO :
MI->implicit_operands()) {
2091 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) {
2102 auto IsHazardFn = [SDSTReg, TRI](
const MachineInstr &
I) {
2107 if (TII->isSALU(
MI)) {
2108 switch (
MI.getOpcode()) {
2109 case AMDGPU::S_SETVSKIP:
2110 case AMDGPU::S_VERSION:
2111 case AMDGPU::S_WAITCNT_VSCNT:
2112 case AMDGPU::S_WAITCNT_VMCNT:
2113 case AMDGPU::S_WAITCNT_EXPCNT:
2116 case AMDGPU::S_WAITCNT_LGKMCNT:
2118 return (
MI.getOperand(1).getImm() == 0) &&
2119 (
MI.getOperand(0).
getReg() == AMDGPU::SGPR_NULL);
2120 case AMDGPU::S_WAITCNT: {
2121 const int64_t
Imm =
MI.getOperand(0).getImm();
2122 AMDGPU::Waitcnt Decoded =
2129 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
2130 "unexpected wait count instruction");
2132 if (TII->isSOPP(
MI))
2148 std::numeric_limits<int>::max())
2152 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
2157bool GCNHazardRecognizer::fixVcmpxExecWARHazard(
MachineInstr *
MI) {
2158 if (!ST.hasVcmpxExecWARHazard())
2160 assert(!ST.hasExtendedWaitCounts());
2165 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2166 if (!
MI->modifiesRegister(AMDGPU::EXEC, TRI))
2172 return I.readsRegister(AMDGPU::EXEC, TRI);
2175 const SIInstrInfo *TII = ST.getInstrInfo();
2176 auto IsExpiredFn = [TII, TRI](
const MachineInstr &
MI, int) {
2178 if (TII->getNamedOperand(
MI, AMDGPU::OpName::sdst))
2180 for (
auto MO :
MI.implicit_operands())
2181 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg())))
2184 if (
MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2186 static_cast<unsigned>(
MI.getOperand(0).getImm())) == 0)
2192 std::numeric_limits<int>::max())
2196 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
2203 if (!ST.hasLdsBranchVmemWARHazard())
2208 bool HasLds =
false;
2209 bool HasVmem =
false;
2210 for (
auto &
MBB : MF) {
2211 for (
auto &
MI :
MBB) {
2214 if (HasLds && HasVmem)
2222 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2223 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
2224 !
I.getOperand(1).getImm();
2227bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(
MachineInstr *
MI) {
2228 if (!RunLdsBranchVmemWARHazardFixup)
2231 assert(ST.hasLdsBranchVmemWARHazard());
2232 assert(!ST.hasExtendedWaitCounts());
2234 auto IsHazardInst = [](
const MachineInstr &
MI) {
2242 auto InstType = IsHazardInst(*
MI);
2246 auto IsExpiredFn = [&IsHazardInst](
const MachineInstr &
I, int) {
2250 auto IsHazardFn = [InstType, &IsHazardInst](
const MachineInstr &
I) {
2254 auto IsHazardFn = [InstType, IsHazardInst](
const MachineInstr &
I) {
2255 auto InstType2 = IsHazardInst(
I);
2256 return InstType2 && InstType != InstType2;
2259 auto IsExpiredFn = [InstType, &IsHazardInst](
const MachineInstr &
I, int) {
2260 auto InstType2 = IsHazardInst(
I);
2261 if (InstType == InstType2)
2268 std::numeric_limits<int>::max();
2272 std::numeric_limits<int>::max())
2275 const SIInstrInfo *TII = ST.getInstrInfo();
2277 TII->get(AMDGPU::S_WAITCNT_VSCNT))
2284bool GCNHazardRecognizer::fixLdsDirectVALUHazard(
MachineInstr *
MI) {
2288 const int NoHazardWaitStates = 15;
2289 const MachineOperand *VDST = TII.getNamedOperand(*
MI, AMDGPU::OpName::vdst);
2292 bool VisitedTrans =
false;
2293 auto IsHazardFn = [
this, VDSTReg, &VisitedTrans](
const MachineInstr &
I) {
2298 return I.readsRegister(VDSTReg, &TRI) ||
I.modifiesRegister(VDSTReg, &TRI);
2300 auto IsExpiredFn = [&](
const MachineInstr &
I,
int WaitStates) {
2301 if (WaitStates >= NoHazardWaitStates)
2307 auto GetWaitStatesFn = [](
const MachineInstr &
MI) {
2311 DenseSet<const MachineBasicBlock *> Visited;
2313 std::next(
MI->getReverseIterator()), 0,
2321 MachineOperand *WaitVdstOp =
2322 TII.getNamedOperand(*
MI, AMDGPU::OpName::waitvdst);
2323 WaitVdstOp->
setImm(std::min(
Count, NoHazardWaitStates));
2328bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(
MachineInstr *
MI) {
2332 const MachineOperand *VDST = TII.getNamedOperand(*
MI, AMDGPU::OpName::vdst);
2335 auto IsHazardFn = [
this, VDSTReg](
const MachineInstr &
I) {
2338 return I.readsRegister(VDSTReg, &TRI) ||
I.modifiesRegister(VDSTReg, &TRI);
2340 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2343 auto IsExpiredFn = [
this, LdsdirCanWait](
const MachineInstr &
I, int) {
2346 (
I.getOpcode() == AMDGPU::S_WAITCNT && !
I.getOperand(0).getImm()) ||
2347 (
I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2349 static_cast<unsigned>(
I.getOperand(0).getImm())) == 0) ||
2351 !TII.getNamedOperand(
I, AMDGPU::OpName::waitvsrc)->getImm());
2355 std::numeric_limits<int>::max())
2358 if (LdsdirCanWait) {
2359 TII.getNamedOperand(*
MI, AMDGPU::OpName::waitvsrc)->setImm(0);
2362 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2369bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(
MachineInstr *
MI) {
2370 if (!ST.hasVALUPartialForwardingHazard())
2372 assert(!ST.hasExtendedWaitCounts());
2377 SmallSetVector<Register, 4> SrcVGPRs;
2379 for (
const MachineOperand &Use :
MI->explicit_uses()) {
2380 if (
Use.isReg() && TRI.isVGPR(MF.getRegInfo(),
Use.getReg()))
2385 if (SrcVGPRs.
size() <= 1)
2403 const int Intv1plus2MaxVALUs = 2;
2404 const int Intv3MaxVALUs = 4;
2405 const int IntvMaxVALUs = 6;
2406 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2409 SmallDenseMap<Register, int, 4> DefPos;
2410 int ExecPos = std::numeric_limits<int>::max();
2413 static unsigned getHashValue(
const StateType &State) {
2415 for (
const auto &[
Reg, Pos] : State.DefPos)
2417 return static_cast<unsigned>(
H);
2419 static bool isEqual(
const StateType &
LHS,
const StateType &
RHS) {
2420 return LHS.DefPos ==
RHS.DefPos &&
LHS.ExecPos ==
RHS.ExecPos &&
2428 auto IsHazardFn = [&,
this](StateType &State,
const MachineInstr &
I) {
2430 if (State.VALUs > NoHazardVALUWaitStates)
2436 (
I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2438 static_cast<unsigned>(
I.getOperand(0).getImm())) == 0))
2445 if (!State.DefPos.count(Src) &&
I.modifiesRegister(Src, &TRI)) {
2446 State.DefPos[Src] = State.VALUs;
2451 if (State.ExecPos == std::numeric_limits<int>::max()) {
2452 if (!State.DefPos.empty() &&
I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
2453 State.ExecPos = State.VALUs;
2460 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2468 if (State.ExecPos == std::numeric_limits<int>::max())
2471 int PreExecPos = std::numeric_limits<int>::max();
2472 int PostExecPos = std::numeric_limits<int>::max();
2474 for (
auto Entry : State.DefPos) {
2475 int DefVALUs =
Entry.second;
2476 if (DefVALUs != std::numeric_limits<int>::max()) {
2477 if (DefVALUs >= State.ExecPos)
2478 PreExecPos = std::min(PreExecPos, DefVALUs);
2480 PostExecPos = std::min(PostExecPos, DefVALUs);
2485 if (PostExecPos == std::numeric_limits<int>::max())
2489 int Intv3VALUs = PostExecPos;
2490 if (Intv3VALUs > Intv3MaxVALUs)
2494 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2495 if (Intv2VALUs > Intv1plus2MaxVALUs)
2499 if (PreExecPos == std::numeric_limits<int>::max())
2503 int Intv1VALUs = PreExecPos - State.ExecPos;
2504 if (Intv1VALUs > Intv1plus2MaxVALUs)
2508 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2513 auto UpdateStateFn = [](StateType &State,
const MachineInstr &
MI) {
2519 std::next(
MI->getReverseIterator())))
2523 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2529bool GCNHazardRecognizer::fixVALUTransUseHazard(
MachineInstr *
MI) {
2530 if (!ST.hasVALUTransUseHazard())
2532 assert(!ST.hasExtendedWaitCounts());
2537 SmallSet<Register, 4> SrcVGPRs;
2539 for (
const MachineOperand &Use :
MI->explicit_uses()) {
2540 if (
Use.isReg() && TRI.isVGPR(MF.getRegInfo(),
Use.getReg()))
2554 const int IntvMaxVALUs = 5;
2555 const int IntvMaxTRANS = 1;
2561 static unsigned getHashValue(
const StateType &State) {
2562 return static_cast<unsigned>(
hash_combine(State.VALUs, State.TRANS));
2564 static bool isEqual(
const StateType &
LHS,
const StateType &
RHS) {
2565 return LHS.VALUs ==
RHS.VALUs &&
LHS.TRANS ==
RHS.TRANS;
2572 auto IsHazardFn = [&,
this](StateType &State,
const MachineInstr &
I) {
2574 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2580 (
I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2582 static_cast<unsigned>(
I.getOperand(0).getImm())) == 0))
2588 if (
I.modifiesRegister(Src, &TRI)) {
2596 auto UpdateStateFn = [](StateType &State,
const MachineInstr &
MI) {
2604 std::next(
MI->getReverseIterator())))
2610 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2616bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(
MachineInstr *
MI) {
2617 if (!ST.hasTransCoexecutionHazard() ||
2622 const SIInstrInfo *TII = ST.getInstrInfo();
2623 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2625 auto IsTransHazardFn = [
MI, TII, TRI](
const MachineInstr &
I) {
2630 Register TransDef = TII->getNamedOperand(
I, AMDGPU::OpName::vdst)->getReg();
2631 for (
const MachineOperand &ValuUse :
MI->explicit_uses()) {
2632 if (ValuUse.isReg() && TRI->regsOverlap(TransDef, ValuUse.getReg()))
2638 for (
const MachineOperand &TransUse :
I.explicit_uses()) {
2639 if (TransUse.isReg() &&
MI->modifiesRegister(TransUse.getReg(), TRI))
2650 const int HasVALU = std::numeric_limits<int>::max();
2651 if (::getWaitStatesSince(IsTransHazardFn,
MI,
IsExpiredFn) == HasVALU)
2654 BuildMI(*
MI->getParent(),
MI,
MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
2662 const SIInstrInfo *TII = ST.getInstrInfo();
2663 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2665 auto IsHazardFn = [
MI, TII, TRI,
this](
const MachineInstr &
I) {
2672 TII->getNamedOperand(*
MI, AMDGPU::OpName::src0)->getReg();
2674 TII->getNamedOperand(*
MI, AMDGPU::OpName::src1)->getReg();
2677 TII->getNamedOperand(
I, AMDGPU::OpName::vdst)->getReg();
2679 if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
2680 TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
2689 TII->getNamedOperand(*
MI, AMDGPU::OpName::src2)->getReg();
2690 if (TRI->regsOverlap(PrevDstReg, CurIndex))
2704 std::numeric_limits<int>::max())
2707 BuildMI(*
MI->getParent(),
MI,
MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
2761 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2762 unsigned Category = 0;
2764 unsigned Latency = SchedModel.computeInstrLatency(&
MI);
2769 assert(!IsSWMMAC &&
"no 4-cycle SWMMAC expected");
2773 Category = IsSWMMAC ? 2 : 0;
2776 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2779 assert(IsLowestRateWMMA &&
"latency 32 is not expected");
2789int GCNHazardRecognizer::checkWMMACoexecutionHazards(
MachineInstr *
MI)
const {
2790 if (!ST.hasWMMACoexecutionHazards())
2793 const SIInstrInfo *TII = ST.getInstrInfo();
2802 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2803 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2804 const bool IsWMMA = TII->isXDLWMMA(*
MI);
2805 const bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2806 ArrayRef<int> RequiredWaitStates =
2807 IsWMMA ? ArrayRef<int>(WMMAWaitStates) :
ArrayRef<int>(VALUWaitStates);
2808 const int MaxWaitStates =
2809 IsWMMA ? (IsLowestRateWMMA ? 17 : 9) : (IsLowestRateWMMA ? 16 : 8);
2811 auto WindowFor = [=](
const MachineInstr &
I) -> std::optional<int> {
2812 if (!TII->isXDLWMMA(
I))
2813 return std::nullopt;
2814 if (IsWMMA ? !hasWMMAToWMMARegOverlap(
I, *
MI)
2815 : !hasWMMAToVALURegOverlap(
I, *
MI))
2816 return std::nullopt;
2821 return getMaxVALUWindowDeficit(MaxWaitStates, WindowFor);
2824bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2826 Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
2827 Register A1 = TII.getNamedOperand(
MI, AMDGPU::OpName::src0)->getReg();
2828 Register B1 = TII.getNamedOperand(
MI, AMDGPU::OpName::src1)->getReg();
2831 if (TRI.regsOverlap(D0, A1) || TRI.regsOverlap(D0, B1))
2835 Register Idx1 = TII.getNamedOperand(
MI, AMDGPU::OpName::src2)->getReg();
2836 if (TRI.regsOverlap(D0, Idx1))
2842bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2845 Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
2846 for (
const MachineOperand &ValuUse :
MI.explicit_uses()) {
2847 if (ValuUse.isReg() && TRI.regsOverlap(D0, ValuUse.getReg()))
2852 Register A0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src0)->getReg();
2853 Register B0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src1)->getReg();
2856 const MachineOperand *
Src2 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src2);
2858 WMMARegs.push_back(
Src2->getReg());
2860 for (
const MachineOperand &ValuDef :
MI.defs()) {
2861 Register VDstReg = ValuDef.getReg();
2862 for (
Register WMMAReg : WMMARegs) {
2863 if (TRI.regsOverlap(VDstReg, WMMAReg))
2870bool GCNHazardRecognizer::isCoexecutionHazardFor(
const MachineInstr &
I,
2874 if (!TII.isXDLWMMA(
I))
2878 if (TII.isXDLWMMA(
MI))
2879 return hasWMMAToWMMARegOverlap(
I,
MI);
2881 return hasWMMAToVALURegOverlap(
I,
MI);
2887 bool IncludeSubloops) {
2890 for (MachineBasicBlock *
MBB :
L->getBlocks()) {
2891 if (!IncludeSubloops && MLI->getLoopFor(
MBB) != L)
2893 for (MachineInstr &
I : *
MBB) {
2896 if (isCoexecutionHazardFor(
I, *
MI))
2903bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(
MachineInstr *
MI,
2904 int WaitStatesNeeded) {
2908 MachineLoop *
L = MLI->getLoopFor(
MI->getParent());
2910 ++NumWMMAHoistingBailed;
2915 if (hasWMMAHazardInLoop(L,
MI)) {
2916 ++NumWMMAHoistingBailed;
2921 MachineLoop *TargetLoop =
L;
2923 if (hasWMMAHazardInLoop(Parent,
MI,
false))
2925 TargetLoop = Parent;
2931 ++NumWMMAHoistingBailed;
2935 LLVM_DEBUG(
dbgs() <<
"WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2941 NumWMMANopsHoisted += WaitStatesNeeded;
2945bool GCNHazardRecognizer::fixWMMACoexecutionHazards(
MachineInstr *
MI) {
2946 int WaitStatesNeeded = checkWMMACoexecutionHazards(
MI);
2947 if (WaitStatesNeeded <= 0)
2950 if (ST.getCLOpts().wmma_vnop_hoisting &&
2951 tryHoistWMMAVnopsFromLoop(
MI, WaitStatesNeeded))
2954 emitVNops(*
MI->getParent(),
MI->getIterator(), WaitStatesNeeded);
2958bool GCNHazardRecognizer::fixShift64HighRegBug(
MachineInstr *
MI) {
2959 if (!ST.hasShift64HighRegBug())
2961 assert(!ST.hasExtendedWaitCounts());
2963 switch (
MI->getOpcode()) {
2966 case AMDGPU::V_LSHLREV_B64_e64:
2967 case AMDGPU::V_LSHRREV_B64_e64:
2968 case AMDGPU::V_ASHRREV_I64_e64:
2972 MachineOperand *Amt = TII.getNamedOperand(*
MI, AMDGPU::OpName::src0);
2977 const MachineRegisterInfo &
MRI = MF.getRegInfo();
2979 if (!TRI.isVGPR(
MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2985 assert(ST.needsAlignedVGPRs());
2986 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2990 MachineOperand *
Src1 = TII.getNamedOperand(*
MI, AMDGPU::OpName::src1);
3001 Register DstReg =
MI->getOperand(0).getReg();
3002 if (!
Src1->isReg() ||
Src1->getReg() != DstReg) {
3003 Register DstLo = TRI.getSubReg(DstReg, AMDGPU::sub0);
3011 bool Overlapped =
MI->modifiesRegister(AmtReg, &TRI);
3013 for (MCRegister
Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
3014 : AMDGPU::VGPR_32RegClass) {
3015 if (!
MI->modifiesRegister(
Reg, &TRI) && !
MI->readsRegister(
Reg, &TRI)) {
3021 Register NewAmt = Overlapped ? (
Register)TRI.getSubReg(NewReg, AMDGPU::sub1)
3026 NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0);
3039 runOnInstruction(
BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt)
3046 BuildMI(*
MBB, std::next(
MI->getIterator()),
DL, TII.get(AMDGPU::V_SWAP_B32),
3052 BuildMI(*
MBB, std::next(
MI->getIterator()),
DL, TII.get(AMDGPU::V_SWAP_B32),
3066 MI->getOperand(0).setReg(NewReg);
3067 Src1->setReg(NewReg);
3068 Src1->setIsKill(
false);
3075int GCNHazardRecognizer::checkNSAtoVMEMHazard(
MachineInstr *
MI)
const {
3076 int NSAtoVMEMWaitStates = 1;
3078 if (!ST.hasNSAtoVMEMBug())
3084 const SIInstrInfo *TII = ST.getInstrInfo();
3085 const auto *
Offset = TII->getNamedOperand(*
MI, AMDGPU::OpName::offset);
3093 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
3094 TII->getInstSizeInBytes(
I) >= 16;
3097 return NSAtoVMEMWaitStates - getWaitStatesSince(
IsHazardFn, 1);
3100int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
3102 int FPAtomicToDenormModeWaitStates = 3;
3104 if (!ST.hasFPAtomicToDenormModeHazard())
3106 assert(!ST.hasExtendedWaitCounts());
3108 if (
MI->getOpcode() != AMDGPU::S_DENORM_MODE)
3117 auto IsExpiredFn = [](
const MachineInstr &
MI,
int WaitStates) {
3124 return FPAtomicToDenormModeWaitStates -
3128int GCNHazardRecognizer::checkMAIHazards(
MachineInstr *
MI)
const {
3131 return ST.hasGFX90AInsts() ? checkMAIHazards90A(
MI) : checkMAIHazards908(
MI);
3134int GCNHazardRecognizer::checkMFMAPadding(
MachineInstr *
MI)
const {
3139 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3143 int NeighborMFMALatency = 0;
3144 auto IsNeighboringMFMA = [&NeighborMFMALatency,
3145 this](
const MachineInstr &
MI) {
3149 NeighborMFMALatency = this->getMFMAPipelineWaitStates(
MI);
3153 const int MaxMFMAPipelineWaitStates = 16;
3154 int WaitStatesSinceNeighborMFMA =
3155 getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
3157 int NeighborMFMAPaddingNeeded =
3159 WaitStatesSinceNeighborMFMA;
3161 return std::max(0, NeighborMFMAPaddingNeeded);
3164int GCNHazardRecognizer::checkMAIHazards908(
MachineInstr *
MI)
const {
3165 int WaitStatesNeeded = 0;
3166 unsigned Opc =
MI->getOpcode();
3168 auto IsVALUFn = [](
const MachineInstr &
MI) {
3172 if (
Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) {
3173 const int LegacyVALUWritesVGPRWaitStates = 2;
3174 const int VALUWritesExecWaitStates = 4;
3175 const int MaxWaitStates = 4;
3177 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3178 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
3179 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3181 if (WaitStatesNeeded < MaxWaitStates) {
3182 for (
const MachineOperand &Use :
MI->explicit_uses()) {
3183 const int MaxWaitStates = 2;
3185 if (!
Use.isReg() || !TRI.isVGPR(MF.getRegInfo(),
Use.getReg()))
3188 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3189 getWaitStatesSinceDef(
Use.getReg(), IsVALUFn, MaxWaitStates);
3190 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3192 if (WaitStatesNeeded == MaxWaitStates)
3198 for (
const MachineOperand &
Op :
MI->explicit_operands()) {
3199 if (!
Op.isReg() || !TRI.isAGPR(MF.getRegInfo(),
Op.getReg()))
3202 if (
Op.isDef() &&
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3205 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3206 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3207 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3208 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3209 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3210 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3211 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3212 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3213 const int MaxWaitStates = 18;
3215 unsigned HazardDefLatency = 0;
3217 auto IsOverlappedMFMAFn = [
Reg, &HazardDefLatency,
3218 this](
const MachineInstr &
MI) {
3225 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&
MI));
3226 return TRI.regsOverlap(DstReg,
Reg);
3229 int WaitStatesSinceDef = getWaitStatesSinceDef(
Reg, IsOverlappedMFMAFn,
3231 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3232 int SrcCIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
3233 int OpNo =
Op.getOperandNo();
3234 if (OpNo == SrcCIdx) {
3235 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3236 }
else if (
Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3237 switch (HazardDefLatency) {
3238 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3240 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3242 case 16: [[fallthrough]];
3243 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3246 }
else if (
Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3247 switch (HazardDefLatency) {
3248 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3250 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3252 case 16: [[fallthrough]];
3253 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3258 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3259 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3261 if (WaitStatesNeeded == MaxWaitStates)
3262 return WaitStatesNeeded;
3264 auto IsAccVgprWriteFn = [
Reg,
this](
const MachineInstr &
MI) {
3265 if (
MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3268 return TRI.regsOverlap(
Reg, DstReg);
3271 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3272 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3273 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3274 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3275 if (OpNo == SrcCIdx)
3276 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3277 else if (
Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3278 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3280 WaitStatesNeededForUse = NeedWaitStates -
3281 getWaitStatesSinceDef(
Reg, IsAccVgprWriteFn, MaxWaitStates);
3282 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3284 if (WaitStatesNeeded == MaxWaitStates)
3285 return WaitStatesNeeded;
3288 if (
Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3289 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3290 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3291 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3292 const int MaxWaitStates = 13;
3293 Register DstReg =
MI->getOperand(0).getReg();
3294 unsigned HazardDefLatency = 0;
3296 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3297 this](
const MachineInstr &
MI) {
3300 Register Reg = TII.getNamedOperand(
MI, AMDGPU::OpName::src2)->getReg();
3302 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&
MI));
3303 return TRI.regsOverlap(
Reg, DstReg);
3306 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
3308 switch (HazardDefLatency) {
3309 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3311 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3313 case 16: [[fallthrough]];
3314 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3318 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3319 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3323 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(
MI));
3325 return WaitStatesNeeded;
3354 return NumPasses + 1 + IsGFX950;
3365 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3383 return NumPasses + 2;
3393 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3396int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
3398 constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3399 constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3400 constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3401 constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3402 constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3403 constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3404 constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3405 constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3406 constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3410 if (TII.isXDL(*Reader) && !TII.isXDL(*Writer))
3414 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3415 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3416 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3417 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3418 return ST.hasGFX950Insts()
3419 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3420 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3421 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3422 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3423 return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3428 int NumPasses = TSchedModel.computeInstrLatency(Writer);
3429 if (ST.hasGFX940Insts()) {
3430 if (!TII.isXDL(*Writer))
3433 return TII.isXDL(*Reader)
3435 NumPasses, ST.hasGFX950Insts())
3437 NumPasses, ST.hasGFX950Insts());
3441 switch (NumPasses) {
3443 return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3444 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3446 return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3447 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3449 return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3450 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3456int GCNHazardRecognizer::getMFMAReadWaitStates(
const MachineInstr &Consumer,
3459 bool IsSrcC)
const {
3461 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3462 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3463 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3464 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3465 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3466 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3467 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3468 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3470 bool FullReg =
Producer.getOperand(0).getReg() ==
Reg;
3471 unsigned Opc1 =
Producer.getOpcode();
3472 int NeedWaitStates = 0;
3477 }
else if (FullReg) {
3478 if ((
Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3479 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3480 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3481 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3482 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3483 else if (ST.hasGFX940Insts() &&
3484 TSchedModel.computeInstrLatency(&Producer) == 2)
3485 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3492 std::max(NeedWaitStates,
3493 getMFMAOverlappedSrcCWaitStates(&Consumer, &Producer));
3496 NeedWaitStates = getMFMAOverlappedSrcCWaitStates(&Consumer, &Producer);
3500 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3501 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3502 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3503 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3506 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3507 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3509 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3510 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3511 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3514 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3516 if (ST.hasGFX940Insts()) {
3520 NumPasses, ST.hasGFX950Insts())
3526 switch (NumPasses) {
3528 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3533 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3537 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3541 return NeedWaitStates;
3544int GCNHazardRecognizer::checkMAIHazards90A(
MachineInstr *
MI)
const {
3545 int WaitStatesNeeded = 0;
3546 unsigned Opc =
MI->getOpcode();
3548 auto IsLegacyVALUFn = [](
const MachineInstr &
MI) {
3553 auto IsLegacyVALUNotDotFn = [](
const MachineInstr &
MI) {
3559 return WaitStatesNeeded;
3561 const int VALUWritesExecWaitStates = 4;
3562 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3563 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
3564 VALUWritesExecWaitStates);
3565 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3567 int SrcCIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
3570 for (
const MachineOperand &Use :
MI->explicit_uses()) {
3571 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3572 const int MaxWaitStates =
3574 16, ST.hasGFX950Insts());
3580 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3581 getWaitStatesSinceDef(
Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
3582 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3584 int OpNo =
Use.getOperandNo();
3585 WaitStatesNeeded = std::max(
3587 getMaxWindowDeficit(
3588 MaxWaitStates, [&](
const MachineInstr &
P) -> std::optional<int> {
3590 !TRI.regsOverlap(
P.getOperand(0).getReg(),
Reg))
3591 return std::nullopt;
3592 return getMFMAReadWaitStates(*
MI,
P,
Reg, OpNo == SrcCIdx);
3595 if (WaitStatesNeeded == MaxWaitStates)
3600 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(
MI));
3602 return WaitStatesNeeded;
3605int GCNHazardRecognizer::checkMAILdStHazards(
MachineInstr *
MI)
const {
3607 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3610 int WaitStatesNeeded = 0;
3612 auto IsAccVgprReadFn = [](
const MachineInstr &
MI) {
3613 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3616 for (
const MachineOperand &
Op :
MI->explicit_uses()) {
3617 if (!
Op.isReg() || !TRI.isVGPR(MF.getRegInfo(),
Op.getReg()))
3622 const int AccVgprReadLdStWaitStates = 2;
3623 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3624 const int MaxWaitStates = 2;
3626 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3627 getWaitStatesSinceDef(
Reg, IsAccVgprReadFn, MaxWaitStates);
3628 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3630 if (WaitStatesNeeded == MaxWaitStates)
3631 return WaitStatesNeeded;
3633 auto IsVALUAccVgprRdWrCheckFn = [
Reg,
this](
const MachineInstr &
MI) {
3634 if (
MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3635 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3637 auto IsVALUFn = [](
const MachineInstr &
MI) {
3641 return getWaitStatesSinceDef(
Reg, IsVALUFn, 2 ) <
3642 std::numeric_limits<int>::max();
3645 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3646 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
3647 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3650 return WaitStatesNeeded;
3653int GCNHazardRecognizer::checkPermlaneHazards(
MachineInstr *
MI)
const {
3654 assert(!ST.hasVcmpxPermlaneHazard() &&
3655 "this is a different vcmpx+permlane hazard");
3656 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3657 const SIInstrInfo *TII = ST.getInstrInfo();
3659 auto IsVCmpXWritesExecFn = [TII, TRI](
const MachineInstr &
MI) {
3663 auto IsVALUFn = [](
const MachineInstr &
MI) {
3667 const int VCmpXWritesExecWaitStates = 4;
3668 const int VALUWritesVDstWaitStates = 2;
3669 int WaitStatesNeeded = 0;
3671 for (
const MachineOperand &
Op :
MI->explicit_uses()) {
3672 if (!
Op.isReg() || !TRI->isVGPR(MF.getRegInfo(),
Op.getReg()))
3676 int WaitStatesSinceDef =
3677 VALUWritesVDstWaitStates -
3678 getWaitStatesSinceDef(
Reg, IsVALUFn,
3679 VALUWritesVDstWaitStates);
3680 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesSinceDef);
3681 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3685 int VCmpXHazardWaits =
3686 VCmpXWritesExecWaitStates -
3687 getWaitStatesSince(IsVCmpXWritesExecFn, VCmpXWritesExecWaitStates);
3689 WaitStatesNeeded = std::max(WaitStatesNeeded, VCmpXHazardWaits);
3690 return WaitStatesNeeded;
3698 return NumPasses + 2;
3708 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3718 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3726 return NumPasses + 2;
3729int GCNHazardRecognizer::checkMAIVALUHazards(
MachineInstr *
MI)
const {
3730 if (!ST.hasGFX90AInsts())
3733 auto IsDGEMMFn = [](
const MachineInstr &
MI) ->
bool {
3741 const MachineRegisterInfo &
MRI = MF.getRegInfo();
3743 int WaitStatesNeeded = 0;
3750 auto IsMFMAWriteFn = [&
Reg,
this](
const MachineInstr &
MI) {
3752 !TRI.regsOverlap(
MI.getOperand(0).getReg(),
Reg))
3757 const MachineInstr *
DOT =
nullptr;
3758 auto IsDotWriteFn = [&
Reg, &
DOT,
this](
const MachineInstr &
MI) {
3760 !TRI.regsOverlap(
MI.getOperand(0).getReg(),
Reg))
3766 bool DGEMMAfterVALUWrite =
false;
3767 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite,
this](
const MachineInstr &
MI) {
3770 DGEMMAfterVALUWrite =
true;
3774 if (!TII.isVALU(
MI,
false) || !DGEMMAfterVALUWrite)
3780 int SrcCIdx = AMDGPU::getNamedOperandIdx(
MI->getOpcode(),
3781 AMDGPU::OpName::src2);
3783 if (IsMemOrExport || IsVALU) {
3784 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3785 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3786 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3787 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3788 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3789 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3790 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3791 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3792 const int DotWriteSameDotReadSrcAB = 3;
3793 const int DotWriteDifferentVALURead = 3;
3794 const int DMFMABetweenVALUWriteVMEMRead = 2;
3795 const int MaxWaitStates =
3797 ST.hasGFX950Insts());
3799 for (
const MachineOperand &Use :
MI->explicit_uses()) {
3805 int WaitStatesSinceDef = getWaitStatesSinceDef(
Reg, IsDotWriteFn,
3808 int NeedWaitStates = 0;
3809 if (
DOT->getOpcode() ==
MI->getOpcode()) {
3810 if (&Use - &
MI->getOperand(0) != SrcCIdx)
3811 NeedWaitStates = DotWriteSameDotReadSrcAB;
3813 NeedWaitStates = DotWriteDifferentVALURead;
3816 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3817 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3824 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3825 DGEMMAfterVALUWrite =
false;
3826 if (TRI.isVectorRegister(
MRI,
Reg)) {
3827 int WaitStatesNeededForUse =
3828 DMFMABetweenVALUWriteVMEMRead -
3829 getWaitStatesSinceDef(
Reg, IsDGEMMHazard,
3830 DMFMABetweenVALUWriteVMEMRead);
3832 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3836 auto MFMAWindow = [&](
const MachineInstr &
Producer) {
3837 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3838 int NeedWaitStates = MaxWaitStates;
3841 switch (NumPasses) {
3843 NeedWaitStates = IsMemOrExport
3844 ? DMFMA4x4WriteVgprMemExpReadWaitStates
3845 : DMFMA4x4WriteVgprVALUReadWaitStates;
3851 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3852 : (ST.hasGFX950Insts()
3853 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3854 : DMFMA16x16WriteVgprVALUReadWaitStates);
3859 }
else if (ST.hasGFX940Insts()) {
3863 NumPasses, ST.hasGFX950Insts())
3867 switch (NumPasses) {
3869 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3872 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3875 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3882 assert(NeedWaitStates <= MaxWaitStates &&
3883 "hazard requirement exceeds the scan window");
3884 return NeedWaitStates;
3887 WaitStatesNeeded = std::max(
3889 getMaxWindowDeficit(MaxWaitStates,
3890 [&](
const MachineInstr &
P) -> std::optional<int> {
3891 if (!IsMFMAWriteFn(
P))
3892 return std::nullopt;
3893 return MFMAWindow(
P);
3896 if (WaitStatesNeeded >= MaxWaitStates)
3901 unsigned Opc =
MI->getOpcode();
3902 const int DMFMAToFMA64WaitStates = 2;
3903 if ((
Opc == AMDGPU::V_FMA_F64_e64 ||
3904 Opc == AMDGPU::V_FMAC_F64_e32 ||
Opc == AMDGPU::V_FMAC_F64_e64 ||
3905 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3906 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3907 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3908 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
3909 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3912 if (!IsVALU && !IsMemOrExport)
3913 return WaitStatesNeeded;
3915 for (
const MachineOperand &Def :
MI->defs()) {
3916 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3917 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3918 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3919 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3920 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3921 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3922 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3923 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3924 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3925 const int DotWriteDifferentVALUWrite = 3;
3926 const int MaxWaitStates =
3928 const int MaxWarWaitStates = 15;
3933 int WaitStatesSinceDef = getWaitStatesSinceDef(
Reg, IsDotWriteFn,
3935 if (DOT &&
DOT->getOpcode() !=
MI->getOpcode())
3936 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
3937 WaitStatesSinceDef);
3939 auto MFMAWawWindow = [&](
const MachineInstr &
Producer) {
3940 int NeedWaitStates = MaxWaitStates;
3941 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3944 switch (NumPasses) {
3946 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3950 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3955 }
else if (ST.hasGFX940Insts()) {
3959 NumPasses, ST.hasGFX950Insts())
3962 switch (NumPasses) {
3964 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3967 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3970 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3977 assert(NeedWaitStates <= MaxWaitStates &&
3978 "hazard requirement exceeds the scan window");
3979 return NeedWaitStates;
3982 WaitStatesNeeded = std::max(
3984 getMaxWindowDeficit(MaxWaitStates,
3985 [&](
const MachineInstr &
P) -> std::optional<int> {
3986 if (!IsMFMAWriteFn(
P))
3987 return std::nullopt;
3988 return MFMAWawWindow(
P);
3991 if (WaitStatesNeeded >= MaxWaitStates)
3995 auto SMFMAReadAsCWindow =
3996 [&](
const MachineInstr &Reader) -> std::optional<int> {
4000 return std::nullopt;
4002 if (ST.hasGFX940Insts() && !TII.isXDL(Reader))
4003 return std::nullopt;
4005 const MachineOperand *SrcC =
4006 TII.getNamedOperand(Reader, AMDGPU::OpName::src2);
4007 assert(SrcC &&
"an MFMA must have a src2 operand");
4009 return std::nullopt;
4011 switch (TSchedModel.computeInstrLatency(&Reader)) {
4013 return SMFMA4x4ReadVgprVALUWarWaitStates;
4015 assert(ST.hasGFX940Insts() &&
"4-pass XDL is gfx940 or later");
4016 return GFX940_XDL4PassReadVgprVALUWarWaitStates;
4018 return SMFMA16x16ReadVgprVALUWarWaitStates;
4022 return SMFMA32x32ReadVgprVALUWarWaitStates;
4027 std::max(WaitStatesNeeded,
4028 getMaxWindowDeficit(MaxWarWaitStates, SMFMAReadAsCWindow));
4031 return WaitStatesNeeded;
4044 return MAI !=
nullptr;
4048 if (IsMFMAFn(*
MI)) {
4049 int W = getWaitStatesSince(IsMFMAFn, 16);
4051 return W < (int)TSchedModel.computeInstrLatency(MAI);
4065 while (
I->isBundledWithPred())
4071 if (
I->getOpcode() != AMDGPU::S_GETPC_B64)
4075 const unsigned NewBytes = 4;
4077 "Unexpected instruction insertion in bundle");
4080 while (NextMI !=
End && NextMI->isBundledWithPred()) {
4081 for (
auto &Operand : NextMI->operands()) {
4082 if (Operand.isGlobal())
4083 Operand.setOffset(Operand.getOffset() + NewBytes);
4089bool GCNHazardRecognizer::fixVALUMaskWriteHazard(
MachineInstr *
MI) {
4090 if (!ST.hasVALUMaskWriteHazard())
4092 assert(!ST.hasExtendedWaitCounts());
4099 if (!IsSALU && !IsVALU)
4111 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4112 const MachineRegisterInfo &
MRI = MF.getRegInfo();
4117 case AMDGPU::EXEC_LO:
4118 case AMDGPU::EXEC_HI:
4120 case AMDGPU::SGPR_NULL:
4121 case AMDGPU::SGPR_NULL64:
4129 return Reg == AMDGPU::VCC ||
Reg == AMDGPU::VCC_LO ||
Reg == AMDGPU::VCC_HI;
4133 SmallSet<Register, 2> HazardSGPRs;
4135 static unsigned getHashValue(
const StateType &State) {
4138 static bool isEqual(
const StateType &
LHS,
const StateType &
RHS) {
4139 return LHS.HazardSGPRs ==
RHS.HazardSGPRs;
4143 StateType InitialState;
4146 MachineOperand *HazardDef =
nullptr;
4147 for (MachineOperand &
Op :
MI->all_defs()) {
4149 if (IgnoreableSGPR(
Reg))
4152 if (
Op.isImplicit())
4154 if (!TRI->isSGPRReg(
MRI,
Reg))
4167 if (AMDGPU::SReg_32RegClass.
contains(HazardReg)) {
4168 InitialState.HazardSGPRs.insert(HazardReg);
4171 InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub0));
4172 InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub1));
4175 auto IsHazardFn = [&](StateType &State,
const MachineInstr &
I) {
4176 if (State.HazardSGPRs.empty())
4179 switch (
I.getOpcode()) {
4180 case AMDGPU::V_ADDC_U32_e32:
4181 case AMDGPU::V_ADDC_U32_dpp:
4182 case AMDGPU::V_CNDMASK_B16_t16_e32:
4183 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4184 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4185 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4186 case AMDGPU::V_CNDMASK_B32_e32:
4187 case AMDGPU::V_CNDMASK_B32_dpp:
4188 case AMDGPU::V_DIV_FMAS_F32_e64:
4189 case AMDGPU::V_DIV_FMAS_F64_e64:
4190 case AMDGPU::V_SUBB_U32_e32:
4191 case AMDGPU::V_SUBB_U32_dpp:
4192 case AMDGPU::V_SUBBREV_U32_e32:
4193 case AMDGPU::V_SUBBREV_U32_dpp: {
4197 case AMDGPU::V_ADDC_U32_e64:
4198 case AMDGPU::V_ADDC_U32_e64_dpp:
4199 case AMDGPU::V_CNDMASK_B16_t16_e64:
4200 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4201 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4202 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4203 case AMDGPU::V_CNDMASK_B32_e64:
4204 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4205 case AMDGPU::V_SUBB_U32_e64:
4206 case AMDGPU::V_SUBB_U32_e64_dpp:
4207 case AMDGPU::V_SUBBREV_U32_e64:
4208 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4210 const MachineOperand *SSRCOp = TII.getNamedOperand(
I, AMDGPU::OpName::src2);
4212 bool Result = TRI->regsOverlap(SSRCOp->
getReg(), HazardReg);
4220 auto UpdateStateFn = [&](StateType &State,
const MachineInstr &
I) {
4222 for (
auto &
Op :
I.all_defs()) {
4224 if (IgnoreableSGPR(
Reg))
4227 if (
Op.isImplicit())
4229 if (!TRI->isSGPRReg(
MRI,
Reg))
4236 for (
Register SGPR : State.HazardSGPRs) {
4237 if (
Reg == SGPR || TRI->regsOverlap(
Reg, SGPR))
4241 State.HazardSGPRs.erase(SGPR);
4248 std::next(
MI->getReverseIterator())))
4258 auto NextMI = std::next(
MI->getIterator());
4259 auto NewMI =
BuildMI(*
MI->getParent(), NextMI,
MI->getDebugLoc(),
4260 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
4272 if (EntryMBB.
begin() != EntryMBB.
end()) {
4273 auto &EntryMI = *EntryMBB.
begin();
4274 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4275 EntryMI.getOperand(0).getImm() >= Priority)
4284bool GCNHazardRecognizer::fixRequiredExportPriority(
MachineInstr *
MI) {
4285 if (!ST.hasRequiredExportPriority())
4303 const int MaxPriority = 3;
4304 const int NormalPriority = 2;
4305 const int PostExportPriority = 0;
4307 auto It =
MI->getIterator();
4308 switch (
MI->getOpcode()) {
4309 case AMDGPU::S_ENDPGM:
4310 case AMDGPU::S_ENDPGM_SAVED:
4311 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4312 case AMDGPU::SI_RETURN_TO_EPILOG:
4315 if (MF->getFrameInfo().hasCalls())
4318 case AMDGPU::S_SETPRIO: {
4320 auto &PrioOp =
MI->getOperand(0);
4321 int Prio =
static_cast<int>(PrioOp.getImm());
4322 bool InWA = (Prio == PostExportPriority) &&
4323 (It !=
MBB->
begin() && TII.isEXP(*std::prev(It)));
4324 if (InWA || Prio >= NormalPriority)
4326 PrioOp.setImm(std::min(Prio + NormalPriority, MaxPriority));
4330 if (!TII.isEXP(*
MI))
4341 auto NextMI = std::next(It);
4342 bool EndOfShader =
false;
4343 if (NextMI !=
MBB->
end()) {
4345 if (TII.isEXP(*NextMI))
4348 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4349 NextMI->getOperand(0).getImm() == PostExportPriority)
4351 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4358 .
addImm(PostExportPriority);
4362 BuildMI(*
MBB, NextMI,
DL, TII.get(AMDGPU::S_WAITCNT_EXPCNT))
4363 .
addReg(AMDGPU::SGPR_NULL)
4385 while (
I !=
End &&
I->isMetaInstruction())
4394 assert(ST.hasRequiresInitialUnclausedVmem() &&
4395 "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
4396 "prologue to satisfy the cross-wave entry mitigation");
4407 if (NextI !=
MBB->
end() && TII.isVPermPk16SafeInstr(*NextI))
4413 BuildMI(*
MBB, NextI,
MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4421 const SIInstrInfo *TII = ST.getInstrInfo();
4433 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4438bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(
MachineInstr *
MI) {
4439 if (
MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4442 const SIInstrInfo *TII = ST.getInstrInfo();
4444 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4446 BuildMI(*
MI->getParent(), std::next(
MI->getIterator()),
MI->getDebugLoc(),
4447 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4453bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(
MachineInstr *
MI) {
4459 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4460 const SIInstrInfo *TII = ST.getInstrInfo();
4462 const int FlatScrBaseWaitStates = 10;
4464 bool ReadsFlatScrLo =
4466 bool ReadsFlatScrHi =
4467 MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4473 ReadsFlatScrLo =
true;
4476 ReadsFlatScrHi =
true;
4481 const MachineRegisterInfo &
MRI = MF.getRegInfo();
4484 DenseSet<const MachineBasicBlock *> Visited;
4486 return MI.modifiesRegister(
Reg, TRI);
4491 auto IsSGPRDef = [TII, TRI, &
MRI](
const MachineInstr &
MI) ->
unsigned {
4492 if (!TII->isSALU(
MI) && !TII->isVALU(
MI,
false))
4494 for (
const MachineOperand &MO :
MI.all_defs()) {
4495 if (TRI->isSGPRReg(
MRI, MO.getReg()))
4501 auto IsExpiredFn = [=](
const MachineInstr &
MI,
int SgprWrites) {
4502 if (
MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4503 unsigned Wait =
static_cast<unsigned>(
MI.getOperand(0).
getImm());
4508 return SgprWrites >= FlatScrBaseWaitStates;
4511 return ::getWaitStatesSince(
4512 IsHazardFn,
MI->getParent(), std::next(
MI->getReverseIterator()),
4513 0,
IsExpiredFn, Visited, IsSGPRDef) < FlatScrBaseWaitStates;
4517 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4519 !IsRegDefHazard(AMDGPU::SGPR103)))
4523 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4534 BuildMI(*
MI->getParent(),
MI,
MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4535 BuildMI(*
MI->getParent(),
MI,
MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4540 auto IsTDM = [&](
const MachineInstr &
MI) ->
bool {
4542 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4549 if (
MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4551 return MI.getOperand(0).getImm() <= 10;
4555 std::numeric_limits<int>::max())
4559 TII.get(AMDGPU::S_WAIT_TENSORCNT))
4564bool GCNHazardRecognizer::fixTransF16ToPartialVGPRWriteHazard(
4569 const int NumVALUNeeded = 4;
4570 const MachineRegisterInfo &
MRI = MF.getRegInfo();
4575 static unsigned getHashValue(
const StateType &State) {
return State.VALUs; }
4576 static bool isEqual(
const StateType &
LHS,
const StateType &
RHS) {
4577 return LHS.VALUs ==
RHS.VALUs;
4583 auto IsHazardFn = [&](StateType &State,
const MachineInstr &
I) {
4585 if (State.VALUs >= NumVALUNeeded)
4589 if (
I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4591 static_cast<unsigned>(
I.getOperand(0).getImm())) == 0)
4598 MCRegister
Def =
I.getOperand(0).getReg();
4599 if (!TRI.isVGPR(
MRI, Def))
4602 unsigned Opc =
I.getOpcode();
4603 const MCInstrDesc &
Desc = TII.get(
Opc);
4607 Desc.operands()[AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0)]
4625 if (TRI.getRegSizeInBits(Def,
MRI) == 16) {
4628 ? std::make_pair(AMDGPU::hi16, AMDGPU::lo16)
4629 : std::make_pair(AMDGPU::lo16, AMDGPU::hi16);
4631 TRI.getMatchingSuperReg(Def,
Sub.first, &AMDGPU::VGPR_32RegClass);
4632 MCRegister OffendingDef = TRI.getSubReg(Def32,
Sub.second);
4634 return MI->modifiesRegister(OffendingDef,
nullptr) ?
HazardFound
4639 if (ST.useRealTrue16Insts())
4647 auto UpdateStateFn = [](StateType &State,
const MachineInstr &
MI) {
4653 std::next(
MI->getReverseIterator())))
4657 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
The AMDGPU TargetMachine interface definition for hw codegen targets.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(AttributeSet CallerAttrs, AttributeSet CalleeAttrs)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST)
static bool consumesDstSelForwardingOperand(const MachineInstr *VALU, const MachineOperand *Dst, const SIRegisterInfo *TRI)
Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .
static bool isSGetReg(unsigned Opcode)
static bool breaksSMEMSoftClause(MachineInstr *MI)
static bool isLdsDma(const MachineInstr &MI)
static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses, bool IsGFX950)
static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI, const SIInstrInfo *TII, const TargetSchedModel &SchedModel, const GCNSubtarget &ST)
static bool isRFE(unsigned Opcode)
static bool isRWLane(unsigned Opcode)
static bool isSMovRel(unsigned Opcode)
static unsigned getMFMANonMacAGPRFormOp(unsigned Opc)
One MFMA can be written with up to four opcodes that differ only in how vdst and src2 are encoded: bo...
#define DEBUG_TYPE_VERBOSE
static const MachineOperand * getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST)
Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value i...
static int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses, bool IsGFX950)
static void updateGetPCBundle(MachineInstr *NewMI)
static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses, bool IsGFX950)
static bool isStoreCountWaitZero(const MachineInstr &I)
static bool breaksVMEMSoftClause(MachineInstr *MI)
static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool isSSetReg(unsigned Opcode)
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
static bool isSameMFMA(unsigned Opc0, unsigned Opc1)
static bool isDivFMas(unsigned Opcode)
static unsigned getCountedWaitStates(const MachineInstr &MI, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates)
static bool hasHazard(StateT InitialState, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *InitialMBB, MachineBasicBlock::const_reverse_instr_iterator InitialI)
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses)
static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses, bool IsGFX950)
static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses)
static MachineBasicBlock::iterator skipMetaInstructionsForward(MachineBasicBlock::iterator I, MachineBasicBlock::iterator End)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses)
static bool isCoexecutableVALUInst(const MachineInstr &MI)
static bool ensureEntrySetPrio(MachineFunction *MF, int Priority, const SIInstrInfo &TII)
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred, const MachineBasicBlock *MBB)
static bool isPermlane(const MachineInstr &MI)
static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses)
static int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses, bool IsGFX950)
AMD GCN specific subclass of TargetSubtarget.
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static llvm::Error parse(GsymDataExtractor &Data, uint64_t BaseAddr, LineEntryCallback const &Callback)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
MachineRegisterInfo & MRI
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
static const uint32_t IV[8]
unsigned get(InstCounterType T) const
BitVector & set()
Set all bits in the bitvector.
std::pair< iterator, bool > insert_as(std::pair< KeyT, ValueT > &&KV, const LookupKeyT &Val)
Alternate version of insert() which allows a different, and possibly less expensive,...
Implements a dense probed hash-table based set.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
unsigned getHazardWaitStates(MachineInstr *MI) const
Returns the number of wait states until all hazards for MI are resolved.
unsigned PreEmitNoopsCommon(MachineInstr *) const
OperatingMode
Operating mode for the hazard recognizer.
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
function_ref< bool(const MachineInstr &)> IsHazardFn
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
function_ref< unsigned int(const MachineInstr &)> GetNumWaitStatesFn
bool ShouldPreferAnother(SUnit *SU) const override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
bool hasPhysRegs() const
Returns true if instruction operands are physical registers, so that hazards defined by register depe...
function_ref< bool(const MachineInstr &, int WaitStates)> IsExpiredFn
bool isSchedulerMode() const
Returns true if running as a scheduler (pre-RA or post-RA).
GCNHazardRecognizer(const MachineFunction &MF, OperatingMode Mode, MachineLoopInfo *MLI=nullptr)
Construct with explicit operating mode.
static AMDGPU::CoExecMaskT getCoExecMaskForMI(const MachineInstr &MI, const SIInstrInfo &TII)
Get the CoExecMask for a given instruction.
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
bool isHazardRecognizerMode() const
Returns true if running as the standalone hazard recognizer pass.
bool isPreRA() const
Returns true if running in pre-RA scheduling mode.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
Wrapper class representing physical registers. Should be passed by value.
Instructions::const_reverse_iterator const_reverse_instr_iterator
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Instructions::iterator instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool isBundled() const
Return true if this instruction part of a bundle.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
void setIsKill(bool Val=true)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool isConstantPhysReg(MCRegister PhysReg) const
Returns true if PhysReg is unallocatable and constant throughout the function.
LLVM_ABI bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Wrapper class representing virtual and physical registers.
static bool isDS(const MachineInstr &MI)
static bool isVMEM(const MachineInstr &MI)
static bool isSMRD(const MachineInstr &MI)
static bool isMTBUF(const MachineInstr &MI)
static bool isDGEMM(unsigned Opcode)
static bool isEXP(const MachineInstr &MI)
static bool isSALU(const MachineInstr &MI)
static bool isSDWA(const MachineInstr &MI)
static bool isDOT(const MachineInstr &MI)
static bool usesTENSOR_CNT(const MachineInstr &MI)
static bool isSWMMAC(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isVPermPk16(unsigned Opcode)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
static bool isTRANS(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
static bool isWaitcnt(unsigned Opcode)
static bool isVOP3(const MCInstrDesc &Desc)
static bool isDPP(const MachineInstr &MI)
static bool isMFMA(const MachineInstr &MI)
static bool isMAI(const MCInstrDesc &Desc)
static bool isFPAtomic(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
static bool isVOP3P(const MachineInstr &MI)
static bool isWMMA(const MachineInstr &MI)
static bool isLDSDMA(const MachineInstr &MI)
unsigned getOccupancy() const
Scheduling unit. This is a node in the scheduling DAG.
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
unsigned getMaxLookAhead() const
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
virtual void EmitNoops(unsigned Quantity)
EmitNoops - This callback is invoked when noops were added to the instruction stream.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Provide an instruction scheduling machine model to CodeGen passes.
std::pair< iterator, bool > insert(const ValueT &V)
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
A range adaptor for a pair of iterators.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc)
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst)
unsigned decodeFieldSaSdst(unsigned Encoded)
unsigned decodeFieldVaSdst(unsigned Encoded)
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc)
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst)
unsigned decodeFieldVaVdst(unsigned Encoded)
unsigned decodeFieldVmVsrc(unsigned Encoded)
unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst)
const char * getCoExecMaskName(CoExecMaskT Mask)
Return a human-readable name for a mask holding a single instruction class, as produced by getCoExecM...
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
constexpr unsigned MaxCoExecStages
Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
FPType getFPDstSelType(unsigned Opc)
bool isGFX12Plus(const MCSubtargetInfo &STI)
StringRef getSchedStrategy(const AMDGPUOptions &CLOpts, const Function &F)
const char * getStageTypeName(CoExecStageType T)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
LLVM_READONLY int32_t getMFMAEarlyClobberOp(uint32_t Opcode)
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded)
CoExecInfo getCoExecInfo(const MachineInstr &MI, const SIInstrInfo &TII)
Get co-execution info for a WMMA instruction, selecting the per-cycle slot pattern from the opcode (a...
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
LLVM_READONLY int32_t getAGPRFormOp(uint32_t Opcode)
InstructionFlavor classifyFlavor(const MachineInstr &MI, const SIInstrInfo &SII)
Classify MI into the execution flavor that drives both the scheduler's slot preferences and the hazar...
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_FP16
CoExecMaskT getCoExecMask(InstructionFlavor F)
Map a flavor to the co-execution class it occupies in a window slot.
bool isGFX1250(const MCSubtargetInfo &STI)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
This namespace contains all of the command line option processing machinery.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
NodeAddr< DefNode * > Def
NodeAddr< UseNode * > Use
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
constexpr RegState getDeadRegState(bool B)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
@ Sub
Subtraction of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Co-execution characteristics for a multi-cycle instruction.
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
An information struct used to provide DenseMap with the various necessary components for a given valu...