LLVM 24.0.0git
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUTargetMachine.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "GCNSubtarget.h"
18#include "llvm/ADT/Statistic.h"
23#include "llvm/Support/Debug.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(0, Value))
45 return O.error("'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error("'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61//===----------------------------------------------------------------------===//
62// Hazard Recognizer Implementation
63//===----------------------------------------------------------------------===//
64
66 const GCNSubtarget &ST);
67
70 MachineLoopInfo *MLI)
71 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
72 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
73 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
74 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
75 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
76 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
78 if (isPreRA())
79 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
80 });
81}
82
86
88 // Dump any active co-execution window that did not complete naturally
89 // (e.g. region ended before the window expired).
91 if (CurrentCoExecStage.has_value()) {
92 unsigned Stage = *CurrentCoExecStage;
93 if (Stage < AMDGPU::MaxCoExecStages)
94 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
95 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
96 dumpCoExecWindow();
97 }
98 });
99}
100
102 EmittedInstrs.clear();
103 EmittedVALUInstrs.clear();
104 HasPendingWMMACoexecHazard = false;
105 if (isSchedulerMode())
106 schedulerReset();
107}
108
109void GCNHazardRecognizer::schedulerReset() {
110 LLVM_DEBUG({
111 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
112 CyclesUntilVALU > 0)
113 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
114 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
115 });
116 CurrentCoExecStage = std::nullopt;
117 CyclesUntilTRANS = 0;
118 CyclesUntilVALU = 0;
119 ActiveCoExecInfo = AMDGPU::CoExecInfo();
120 CoExecWindowLog.fill('.');
121}
122
123void GCNHazardRecognizer::dumpCoExecWindow() const {
124 unsigned W = ActiveCoExecInfo.TotalWindow;
125 if (W == 0)
126 return;
127
128 // Print the stage numbers row.
129 dbgs() << " Stages: ";
130 for (unsigned I = 0; I < W; ++I)
131 dbgs() << I % 10 << ' ';
132 dbgs() << '\n';
133
134 // Print the pattern row.
135 dbgs() << " Slots: ";
136 for (unsigned I = 0; I < W; ++I)
137 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
138 dbgs() << '\n';
139
140 // Print the scheduled row.
141 dbgs() << " Scheduled: ";
142 for (unsigned I = 0; I < W; ++I)
143 dbgs() << CoExecWindowLog[I] << ' ';
144 dbgs() << '\n';
145}
146
147void GCNHazardRecognizer::schedulerAdvanceCycle() {
148 // Record what happened at the current stage of the co-exec window.
149 if (CurrentCoExecStage.has_value()) {
150 unsigned Stage = *CurrentCoExecStage;
151 if (Stage < AMDGPU::MaxCoExecStages) {
152 if (CurrCycleInstr)
153 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
154 else
155 CoExecWindowLog[Stage] = '-';
156 }
157 }
158
159 LLVM_DEBUG({
160 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
161 CyclesUntilVALU > 0;
162 if (HasState) {
163 dbgs() << " Scheduler AdvanceCycle:";
164 if (CurrentCoExecStage.has_value()) {
165 unsigned Stage = *CurrentCoExecStage;
166 unsigned Next = Stage + 1;
167 if (Next >= ActiveCoExecInfo.TotalWindow)
168 dbgs() << " stage " << Stage << "->expired";
169 else
170 dbgs() << " stage " << Stage << "->" << Next;
171 }
172 if (CyclesUntilTRANS > 0)
173 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
174 << (CyclesUntilTRANS - 1);
175 if (CyclesUntilVALU > 0)
176 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
177 dbgs() << "\n";
178 }
179 });
180
181 // Decrement hazard counters.
182 if (CyclesUntilTRANS > 0)
183 --CyclesUntilTRANS;
184 if (CyclesUntilVALU > 0)
185 --CyclesUntilVALU;
186
187 // Advance WMMA co-execution window.
188 if (CurrentCoExecStage.has_value()) {
189 unsigned Stage = *CurrentCoExecStage + 1;
190 if (Stage >= ActiveCoExecInfo.TotalWindow) {
191 // Window expired.
192 LLVM_DEBUG({
193 dbgs() << " CoExec window complete:\n";
194 dumpCoExecWindow();
195 });
196 CurrentCoExecStage = std::nullopt;
197 } else {
198 CurrentCoExecStage = Stage;
199 }
200 }
201}
202
203bool GCNHazardRecognizer::hasCoExecWindowModel() const {
204 // The co-execution slot patterns returned by getCoExecInfo() are derived from
205 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
206 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
207 // but have different WMMA latencies, so they need their own slot patterns
208 // before they can be modeled here.
209 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
211 return true;
212
213 if (ST.hasGFX950Insts() &&
214 AMDGPU::getSchedStrategy(ST.getCLOpts(), MF.getFunction()) == "coexec")
215 return true;
216
217 return false;
218}
219
220void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
221 if (!hasCoExecWindowModel())
222 return;
223
226 return;
227
228 // If a previous window was still active, dump it before starting a new one.
229 // Record the current stage (filled by this new WMMA) before dumping.
230 LLVM_DEBUG({
231 if (CurrentCoExecStage.has_value()) {
232 unsigned Stage = *CurrentCoExecStage;
233 if (Stage < AMDGPU::MaxCoExecStages)
234 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
235 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
236 dumpCoExecWindow();
237 }
238 });
239
240 // Start a new co-execution window.
241 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
242 CurrentCoExecStage = 0;
243 CoExecWindowLog.fill('.');
244
245 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
246 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
247 << " " << MI);
248}
249
250void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
251 if (!hasCoExecWindowModel())
252 return;
254 return;
255
256 // Back-to-back TRANS instructions have a 1-cycle hazard.
257 // This is checked via checkTRANSHazard() and does not create a co-exec
258 // window. The TRANS shadow slot allows anything except TRANS and
259 // multi-cycle VALU.
260 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
261 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
262 // the counter is observed at 1 there. That 1-cycle stall lets the
263 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
264 // shadow slot.
265 CyclesUntilTRANS = 2;
266 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
267}
268
269void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
270 if (!hasCoExecWindowModel())
271 return;
272 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
273 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
274 return;
275
276 // Skip WMMA, MFMA, and TRANS - they have their own tracking.
279 return;
280
281 unsigned RepeatRate = TII.getRepeatRate(MI);
282 if (RepeatRate > 1) {
283 // bumpCycle's AdvanceCycle decrements once before the next pick's
284 // hazard check (same convention as CyclesUntilTRANS), so to expose
285 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
286 CyclesUntilVALU = RepeatRate;
287 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
288 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
289 }
290}
291
297
298unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
299 if (!CyclesUntilTRANS)
300 return 0;
301
302 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
304 return CyclesUntilTRANS;
305
306 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
308 TII.getRepeatRate(MI) > 1)
309 return CyclesUntilTRANS;
310
311 return 0;
312}
313
314unsigned
315GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
316 if (!CyclesUntilVALU)
317 return 0;
318
319 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
320 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
321 // SALU, control) can fill the shadow.
322 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
325 return 0;
326
327 return CyclesUntilVALU;
328}
329
330unsigned
331GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
332 // No hazard if not in a WMMA window.
333 if (!CurrentCoExecStage.has_value())
334 return 0;
335
336 unsigned Stage = *CurrentCoExecStage;
338 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
339
340 // No stall required if the instruction can co-execute at the current stage.
341 if (StallCycles == 0)
342 return 0;
343
344 // Stall for the required number of cycles until the next allowed stage.
345 unsigned NextStage = Stage + StallCycles;
346 if (NextStage < ActiveCoExecInfo.TotalWindow) {
349 dbgs() << " CoExec stall: stage=" << Stage << "("
350 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
351 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
352 << " -> stall " << StallCycles << " (next allowed=" << NextStage
353 << ")\n"
354 << " " << MI);
355 return StallCycles;
356 }
357
358 // No compatible slot in window - stall until window ends.
361 dbgs() << " CoExec stall: stage=" << Stage << "("
362 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
363 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
364 << StallCycles << " (window ends)\n"
365 << " " << MI);
366 return StallCycles;
367}
368
369unsigned
370GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
371 // This models a VALU caught in both a WMMA and a TRANS shadow.
372 if (!hasCoExecWindowModel())
373 return 0;
374
375 // No hazard if not in a WMMA window.
376 if (!CurrentCoExecStage.has_value())
377 return 0;
378
379 if (!CyclesUntilTRANS)
380 return 0;
381
382 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
383 return 0;
384
385 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
386 // We need to wait for at least one to clear.
387
388 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
390 return CyclesUntilTRANS +
391 ActiveCoExecInfo.getStallCycles(InstMask, LookAheadStage);
392}
393
394void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
395 LLVM_DEBUG({
396 bool InWindow = CurrentCoExecStage.has_value();
397 bool HasActiveState =
398 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
399 if (HasActiveState) {
400 if (InWindow) {
401 unsigned Stage = *CurrentCoExecStage;
402 dbgs() << " Stage " << Stage << "("
403 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
404 << ") Emit ["
406 << "]: " << *MI;
407 } else {
408 dbgs() << " Emit ["
410 << "]: " << *MI;
411 }
412 }
413 });
415 bool HasActiveState = CurrentCoExecStage.has_value() ||
416 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
417 if (!HasActiveState)
418 dbgs() << " Emit ["
420 << "]: " << *MI;
421 });
422 updateWMMAWindowState(*MI);
423 updateTRANSState(*MI);
424 updateMultiCycleVALUState(*MI);
425}
426
430
432 CurrCycleInstr = MI;
433 if (isSchedulerMode())
434 schedulerEmitInstruction(MI);
435}
436
437static bool isDivFMas(unsigned Opcode) {
438 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
439}
440
441static bool isSGetReg(unsigned Opcode) {
442 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
443}
444
445static bool isSSetReg(unsigned Opcode) {
446 switch (Opcode) {
447 case AMDGPU::S_SETREG_B32:
448 case AMDGPU::S_SETREG_B32_mode:
449 case AMDGPU::S_SETREG_IMM32_B32:
450 case AMDGPU::S_SETREG_IMM32_B32_mode:
451 return true;
452 }
453 return false;
454}
455
456static bool isRWLane(unsigned Opcode) {
457 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
458}
459
460static bool isRFE(unsigned Opcode) {
461 return Opcode == AMDGPU::S_RFE_B64;
462}
463
464static bool isSMovRel(unsigned Opcode) {
465 switch (Opcode) {
466 case AMDGPU::S_MOVRELS_B32:
467 case AMDGPU::S_MOVRELS_B64:
468 case AMDGPU::S_MOVRELD_B32:
469 case AMDGPU::S_MOVRELD_B64:
470 return true;
471 default:
472 return false;
473 }
474}
475
477 const MachineInstr &MI) {
478 if (TII.isAlwaysGDS(MI.getOpcode()))
479 return true;
480
481 switch (MI.getOpcode()) {
482 case AMDGPU::S_SENDMSG:
483 case AMDGPU::S_SENDMSGHALT:
484 case AMDGPU::S_TTRACEDATA:
485 return true;
486 // These DS opcodes don't support GDS.
487 case AMDGPU::DS_NOP:
488 case AMDGPU::DS_PERMUTE_B32:
489 case AMDGPU::DS_BPERMUTE_B32:
490 return false;
491 default:
492 if (TII.isDS(MI.getOpcode())) {
493 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
494 AMDGPU::OpName::gds);
495 if (MI.getOperand(GDS).getImm())
496 return true;
497 }
498 return false;
499 }
500}
501
502static bool isPermlane(const MachineInstr &MI) {
503 unsigned Opcode = MI.getOpcode();
504 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
505 Opcode == AMDGPU::V_PERMLANE64_B32 ||
506 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
507 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
508 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
509 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
510 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
511 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
512 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
513 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
514 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
515 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
517 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
518}
519
520static bool isLdsDma(const MachineInstr &MI) {
522}
523
524static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
525 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
526 AMDGPU::OpName::simm16);
527 return std::get<0>(AMDGPU::Hwreg::HwregEncoding::decode(RegOp->getImm()));
528}
529
532 MachineInstr *MI = SU->getInstr();
533 // If we are not in "HazardRecognizerMode" and therefore not being run from
534 // the scheduler, track possible stalls from hazards but don't insert noops.
536
537 if (MI->isBundle())
538 return NoHazard;
539
540 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
541 if (isSchedulerMode()) {
542 if (checkMultiShadowHazard(*MI) > 0)
543 return Hazard;
544 if (checkWMMACoexecSlot(*MI) > 0)
545 return Hazard;
546 if (checkTRANSHazard(*MI) > 0)
547 return Hazard;
548 if (checkMultiCycleVALUHazard(*MI) > 0)
549 return Hazard;
550 // The remaining checks are all defined by register dependences.
551 if (!hasPhysRegs())
552 return NoHazard;
553 }
554
555 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
556 return HazardType;
557
558 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
559 return HazardType;
560
561 if (checkFPAtomicToDenormModeHazard(MI) > 0)
562 return HazardType;
563
564 // Hazards which cannot be mitigated with S_NOPs.
565 if (!isHazardRecognizerMode()) {
566 if (checkWMMACoexecutionHazards(MI) > 0) {
567 HasPendingWMMACoexecHazard = true;
568 return Hazard;
569 }
570 }
571
572 if (ST.hasNoDataDepHazard())
573 return NoHazard;
574
575 if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
576 return HazardType;
577
578 if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) &&
579 checkVALUHazards(MI) > 0)
580 return HazardType;
581
582 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
583 return HazardType;
584
585 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
586 return HazardType;
587
588 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
589 return HazardType;
590
591 if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false) ||
594 checkMAIVALUHazards(MI) > 0)
595 return HazardType;
596
597 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
598 return HazardType;
599
600 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
601 return HazardType;
602
603 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
604 return HazardType;
605
606 if (((ST.hasReadM0MovRelInterpHazard() &&
607 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
608 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
609 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
610 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
611 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
612 (ST.hasReadM0LdsDirectHazard() &&
613 MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
614 checkReadM0Hazards(MI) > 0)
615 return HazardType;
616
617 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
618 return HazardType;
619
621 checkMAILdStHazards(MI) > 0)
622 return HazardType;
623
624 if (ST.hasPermlaneForwardingHazard() && isPermlane(*MI) &&
625 checkPermlaneHazards(MI) > 0)
626 return HazardType;
627
628 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
629 return HazardType;
630
631 return NoHazard;
632}
633
635 unsigned Quantity) {
636 while (Quantity > 0) {
637 unsigned Arg = std::min(Quantity, 8u);
638 Quantity -= Arg;
639 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
640 .addImm(Arg - 1);
641 }
642}
643
644unsigned
645GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
646 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
647 assert(TSchedModel.getWriteProcResBegin(SC) !=
648 TSchedModel.getWriteProcResEnd(SC));
649 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
650}
651
652void GCNHazardRecognizer::processBundle() {
653 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
654 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
655 // Check bundled MachineInstr's for hazards.
656 for (; MI != E && MI->isInsideBundle(); ++MI) {
657 CurrCycleInstr = &*MI;
658 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
659
661 fixHazards(CurrCycleInstr);
662
663 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
664 }
665
666 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
667 // include the bundled MI directly after, only add a maximum of
668 // (MaxLookAhead - 1) noops to EmittedInstrs.
669 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
670 EmittedInstrs.push_front(nullptr);
671
672 EmittedInstrs.push_front(CurrCycleInstr);
673 EmittedInstrs.resize(MaxLookAhead);
674 }
675 CurrCycleInstr = nullptr;
676}
677
678void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
680
681 unsigned NumPreNoops = PreEmitNoops(MI);
682 EmitNoops(NumPreNoops);
683 if (MI->isInsideBundle())
684 insertNoopsInBundle(MI, TII, NumPreNoops);
685 else
686 TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI),
687 NumPreNoops);
689 AdvanceCycle();
690}
691
694 CurrCycleInstr = MI;
695 unsigned W = PreEmitNoopsCommon(MI);
696 fixHazards(MI);
697 CurrCycleInstr = nullptr;
698 return std::max(W, ST.getCLOpts().snop_padding);
699}
700
702 unsigned W = 0;
703
704 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
705 if (isSchedulerMode()) {
706 W = checkWMMACoexecSlot(*MI);
707 W = std::max(W, checkTRANSHazard(*MI));
708 W = std::max(W, checkMultiCycleVALUHazard(*MI));
709 W = std::max(W, checkMultiShadowHazard(*MI));
710 // The remaining checks are all defined by register dependences.
711 if (!hasPhysRegs())
712 return W;
713 }
714
715 return std::max(W, PreEmitNoopsCommon(MI));
716}
717
719 if (MI->isBundle())
720 return 0;
721
722 int WaitStates = 0;
723
725 return std::max(WaitStates, checkSMRDHazards(MI));
726
727 if (ST.hasNSAtoVMEMBug())
728 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
729
730 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
731
732 if (ST.hasNoDataDepHazard())
733 return WaitStates;
734
736 WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
737
738 if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
739 WaitStates = std::max(WaitStates, checkVALUHazards(MI));
740
742 WaitStates = std::max(WaitStates, checkDPPHazards(MI));
743
744 if (isDivFMas(MI->getOpcode()))
745 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
746
747 if (isRWLane(MI->getOpcode()))
748 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
749
750 if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false) ||
753 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
754
755 if (MI->isInlineAsm())
756 return std::max(WaitStates, checkInlineAsmHazards(MI));
757
758 if (isSGetReg(MI->getOpcode()))
759 return std::max(WaitStates, checkGetRegHazards(MI));
760
761 if (isSSetReg(MI->getOpcode()))
762 return std::max(WaitStates, checkSetRegHazards(MI));
763
764 if (isRFE(MI->getOpcode()))
765 return std::max(WaitStates, checkRFEHazards(MI));
766
767 if ((ST.hasReadM0MovRelInterpHazard() &&
768 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
769 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
770 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
771 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
772 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
773 (ST.hasReadM0LdsDirectHazard() &&
774 MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
775 return std::max(WaitStates, checkReadM0Hazards(MI));
776
778 return std::max(WaitStates, checkMAIHazards(MI));
779
781 return std::max(WaitStates, checkMAILdStHazards(MI));
782
783 if (ST.hasPermlaneForwardingHazard() && isPermlane(*MI))
784 return std::max(WaitStates, checkPermlaneHazards(MI));
785
786 return WaitStates;
787}
788
790 EmittedInstrs.push_front(nullptr);
791}
792
794 if (isSchedulerMode())
795 schedulerAdvanceCycle();
796
797 // When the scheduler detects a stall, it will call AdvanceCycle() without
798 // emitting any instructions.
799 if (!CurrCycleInstr) {
800 EmittedInstrs.push_front(nullptr);
801
802 if (HasPendingWMMACoexecHazard)
803 EmittedVALUInstrs.push_front(nullptr);
804 return;
805 }
806
807 HasPendingWMMACoexecHazard = false;
808
809 if (CurrCycleInstr->isBundle()) {
810 processBundle();
811 return;
812 }
813
814 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
815 if (!NumWaitStates) {
816 CurrCycleInstr = nullptr;
817 return;
818 }
819
820 // Keep track of emitted instructions
821 EmittedInstrs.push_front(CurrCycleInstr);
822
823 bool IsVALUOrWMMA =
824 SIInstrInfo::isVALU(*CurrCycleInstr, /*AllowLDSDMA=*/true) ||
825 SIInstrInfo::isWMMA(*CurrCycleInstr) ||
826 SIInstrInfo::isSWMMAC(*CurrCycleInstr);
827 if (IsVALUOrWMMA) {
828 EmittedVALUInstrs.push_front(CurrCycleInstr);
829 } else {
830 // A pending WMMA co-execution hazard optimistically records stall cycles as
831 // future V_NOPs. If the scheduler instead stalls for a different
832 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
833 // they will not resolve the VALU-pipe hazard, so drop them here.
834 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
835 EmittedVALUInstrs.pop_front();
836 }
837
838 // Add a nullptr for each additional wait state after the first. Make sure
839 // not to add more than getMaxLookAhead() items to the list, since we
840 // truncate the list to that size right after this loop.
841 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
842 i < e; ++i) {
843 EmittedInstrs.push_front(nullptr);
844 }
845
846 // getMaxLookahead() is the largest number of wait states we will ever need
847 // to insert, so there is no point in keeping track of more than that many
848 // wait states.
849 EmittedInstrs.resize(getMaxLookAhead());
850 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
851 EmittedVALUInstrs.resize(MaxVALULookAhead);
852
853 CurrCycleInstr = nullptr;
854}
855
858 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
859}
860
861//===----------------------------------------------------------------------===//
862// Helper Functions
863//===----------------------------------------------------------------------===//
864
866
867// Search for a hazard in a block and its predecessors.
868template <typename StateT>
869static bool
870hasHazard(StateT InitialState,
871 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
872 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
873 const MachineBasicBlock *InitialMBB,
875 struct StateMapKey {
877 size_t Idx;
878 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
879 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
880 }
881 };
882 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
883 static unsigned getHashValue(const StateMapKey &Key) {
884 return StateT::getHashValue((*Key.States)[Key.Idx]);
885 }
886 static unsigned getHashValue(const StateT &State) {
887 return StateT::getHashValue(State);
888 }
889 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
890 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
891 }
892 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
893 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
894 }
895 };
896
899
901 const MachineBasicBlock *MBB = InitialMBB;
902 StateT State = InitialState;
903
905 unsigned WorkIdx = 0;
906 for (;;) {
907 bool Expired = false;
908 for (auto E = MBB->instr_rend(); I != E; ++I) {
909 // No need to look at parent BUNDLE instructions.
910 if (I->isBundle())
911 continue;
912
913 auto Result = IsHazard(State, *I);
914 if (Result == HazardFound)
915 return true;
916 if (Result == HazardExpired) {
917 Expired = true;
918 break;
919 }
920
921 if (I->isInlineAsm() || I->isMetaInstruction())
922 continue;
923
924 UpdateState(State, *I);
925 }
926
927 if (!Expired) {
928 size_t StateIdx = States.size();
929 StateMapKey Key = {&States, StateIdx};
930 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
931 if (Insertion.second) {
932 States.emplace_back(State);
933 } else {
934 StateIdx = Insertion.first->second;
935 }
936 for (MachineBasicBlock *Pred : MBB->predecessors())
937 Worklist.insert(std::pair(Pred, StateIdx));
938 }
939
940 if (WorkIdx == Worklist.size())
941 break;
942
943 size_t StateIdx;
944 std::tie(MBB, StateIdx) = Worklist[WorkIdx++];
945 State = States[StateIdx];
946 I = MBB->instr_rbegin();
947 }
948
949 return false;
950}
951
952// Number of trailing terminators of \p Pred not executed on the edge to \p MBB:
953// control leaves at the first terminator branching to \p MBB, so the ones after
954// it are dead on that edge. A fallthrough executes them all.
956 const MachineBasicBlock *MBB) {
957 const SIInstrInfo &TII =
958 *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo();
959 unsigned Unexecuted = 0, Passed = 0;
960 for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend();
961 I != E && I->isTerminator(); ++I, ++Passed) {
962 // Only an explicit MBB destination identifies this edge; the last match is
963 // the earliest such branch in program order.
964 if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(0).isMBB() &&
965 TII.getBranchDestBlock(*I) == MBB)
966 Unexecuted = Passed;
967 }
968 return Unexecuted;
969}
970
971// Wait states \p MI adds to a distance. Inline asm may be empty, so it adds
972// none, which is the worst case.
973static unsigned
976 return MI.isInlineAsm() ? 0 : GetNumWaitStates(MI);
977}
978
979namespace {
980
981// Answers GCNHazardRecognizer::getMax*WindowDeficit, which documents the
982// result, by walking the CFG backwards from the instruction being checked,
983// counting distances with GetNumWaitStates. That needs the instruction in its
984// final place, so hazard recognizer mode only. Each object answers one query.
985class WindowDeficitSearch {
986public:
987 WindowDeficitSearch(
988 function_ref<std::optional<int>(const MachineInstr &)> WindowFor,
989 int MaxWindow,
992 : WindowFor(WindowFor), GetNumWaitStates(GetNumWaitStates),
993 MaxWindow(MaxWindow) {}
994
995 int run(const MachineInstr &MI);
996
997private:
998 // A block to scan upwards from Start, which is Distance away from the
999 // instruction being checked. In a predecessor, Start is the last instruction
1000 // above the terminators, and Distance includes the ones the edge executes.
1001 struct Arrival {
1002 const MachineBasicBlock *MBB;
1004 int Distance;
1005 };
1006
1007 void arrive(const MachineBasicBlock *MBB,
1009 int Distance);
1010 void scan(Arrival A);
1011
1012 const function_ref<std::optional<int>(const MachineInstr &)> WindowFor;
1013 const GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates;
1014 const int MaxWindow;
1015 int Deficit = 0;
1016 // Pending[D] holds the arrivals at distance D, and Best the smallest distance
1017 // each block has been queued at. Buckets are emptied in increasing distance,
1018 // so the first arrival scanned at a block is the shortest way into it, and
1019 // any later one is stale.
1021 DenseMap<const MachineBasicBlock *, int> Best;
1022};
1023
1024} // end anonymous namespace
1025
1026void WindowDeficitSearch::arrive(
1027 const MachineBasicBlock *MBB,
1029 // No window exceeds MaxWindow, so nothing this far back can beat the deficit
1030 // already found.
1031 if (MaxWindow - Distance <= Deficit)
1032 return;
1033
1034 if (Pending.empty())
1035 Pending.resize(MaxWindow);
1036
1037 auto [It, Inserted] = Best.try_emplace(MBB, Distance);
1038 if (!Inserted) {
1039 if (It->second <= Distance)
1040 return;
1041 It->second = Distance;
1042 }
1043
1044 assert(Distance < MaxWindow && "arrival outside the window");
1045 Pending[Distance].push_back({MBB, Start, Distance});
1046}
1047
1048// \p A is a copy because arrive() may append to the bucket it came from, which
1049// can move that bucket's elements.
1050void WindowDeficitSearch::scan(Arrival A) {
1051 // A shorter way into this block has been queued since.
1052 if (Best.lookup(A.MBB) < A.Distance)
1053 return;
1054
1055 int Distance = A.Distance;
1056 for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) {
1057 if (I->isBundle())
1058 continue;
1059
1060 if (std::optional<int> Window = WindowFor(*I)) {
1061 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1062 Deficit = std::max(Deficit, *Window - Distance);
1063 }
1064
1065 Distance += getCountedWaitStates(*I, GetNumWaitStates);
1066
1067 if (Distance >= MaxWindow)
1068 return;
1069 }
1070
1071 for (const MachineBasicBlock *Pred : A.MBB->predecessors()) {
1072 auto Start = Pred->instr_rbegin(), End = Pred->instr_rend();
1073 for (unsigned N = countUnexecutedTerminators(*Pred, A.MBB); N; --N)
1074 ++Start;
1075
1076 int Executed = 0;
1077 for (; Start != End && Start->isTerminator(); ++Start) {
1078 assert(!WindowFor(*Start) && "terminators must not ask for a window");
1079 Executed += getCountedWaitStates(*Start, GetNumWaitStates);
1080 }
1081
1082 arrive(Pred, Start, Distance + Executed);
1083 }
1084}
1085
1086int WindowDeficitSearch::run(const MachineInstr &MI) {
1087 // Only the part above MI is scanned here, so the block stays out of Best: a
1088 // backedge into it must still scan it from its end.
1089 scan({MI.getParent(), std::next(MI.getReverseIterator()), 0});
1090
1091 // Distances never shrink along a path, so a scan only queues arrivals at the
1092 // distance being swept or further, and one pass over the buckets is enough.
1093 // The inner loop re-reads the size because an edge that adds nothing appends
1094 // to the bucket in hand; Best keeps a cycle that adds nothing from looping.
1095 for (int Distance = 0; Distance < static_cast<int>(Pending.size());
1096 ++Distance)
1097 for (unsigned I = 0; I != Pending[Distance].size(); ++I)
1098 scan(Pending[Distance][I]);
1099
1100 return Deficit;
1101}
1102
1103// Returns a minimum wait states since \p I walking all predecessors.
1104// Only scans until \p IsExpired does not return true.
1105// Can only be run in a hazard recognizer mode.
1106static int
1108 const MachineBasicBlock *MBB,
1110 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
1114 for (auto E = MBB->instr_rend(); I != E; ++I) {
1115 // Don't add WaitStates for parent BUNDLE instructions.
1116 if (I->isBundle())
1117 continue;
1118
1119 if (I->isMetaInstruction())
1120 continue;
1121
1122 if (IsHazard(*I))
1123 return WaitStates;
1124
1125 if (I->isInlineAsm())
1126 continue;
1127
1128 WaitStates += GetNumWaitStates(*I);
1129
1130 if (IsExpired(*I, WaitStates))
1131 return std::numeric_limits<int>::max();
1132 }
1133
1134 int MinWaitStates = std::numeric_limits<int>::max();
1135 for (MachineBasicBlock *Pred : MBB->predecessors()) {
1136 if (!Visited.insert(Pred).second)
1137 continue;
1138
1139 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates,
1140 IsExpired, Visited, GetNumWaitStates);
1141
1142 MinWaitStates = std::min(MinWaitStates, W);
1143 }
1144
1145 return MinWaitStates;
1146}
1147
1148static int
1150 const MachineInstr *MI,
1155 return getWaitStatesSince(IsHazard, MI->getParent(),
1156 std::next(MI->getReverseIterator()), 0, IsExpired,
1157 Visited, GetNumWaitStates);
1158}
1159
1160int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow,
1161 WindowForFn WindowFor) const {
1163 return WindowDeficitSearch(WindowFor, MaxWindow).run(*CurrCycleInstr);
1164
1165 // EmittedInstrs is capped and can be shorter than the widest window, which
1166 // costs scheduling quality only: the standalone pass still pads.
1167 int Deficit = 0, Distance = 0;
1168 for (MachineInstr *MI : EmittedInstrs) {
1169 if (MI) {
1170 if (std::optional<int> Window = WindowFor(*MI)) {
1171 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1172 Deficit = std::max(Deficit, *Window - Distance);
1173 }
1174 }
1175
1176 Distance +=
1178
1179 if (Distance >= MaxWindow)
1180 break;
1181 }
1182 return Deficit;
1183}
1184
1185int GCNHazardRecognizer::getWaitStatesSince(
1186 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1187 if (isHazardRecognizerMode()) {
1188 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1189 return WaitStates >= Limit;
1190 };
1191 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn,
1192 GetNumWaitStates);
1193 }
1194
1195 int WaitStates = 0;
1196 for (MachineInstr *MI : EmittedInstrs) {
1197 if (MI) {
1198 if (IsHazard(*MI))
1199 return WaitStates;
1200
1201 if (MI->isInlineAsm())
1202 continue;
1203 }
1204 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1205
1206 if (WaitStates >= Limit)
1207 break;
1208 }
1209 return std::numeric_limits<int>::max();
1210}
1211
1212int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1213 int Limit) const {
1214 return getWaitStatesSince(IsHazard, Limit, SIInstrInfo::getNumWaitStates);
1215}
1216
1217int GCNHazardRecognizer::getMaxVALUWindowDeficit(int MaxWindow,
1218 WindowForFn WindowFor) const {
1219 if (isHazardRecognizerMode()) {
1220 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1221 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1222 };
1223 return WindowDeficitSearch(WindowFor, MaxWindow, GetVALUWaitStates)
1224 .run(*CurrCycleInstr);
1225 }
1226
1227 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a MaxWindow beyond that
1228 // window could miss a hazard. Keep the cap in sync with the wait-state
1229 // tables.
1230 assert(MaxWindow <= static_cast<int>(MaxVALULookAhead) &&
1231 "MaxWindow exceeds the EmittedVALUInstrs lookahead window");
1232 int WaitStates = 0;
1233 for (MachineInstr *MI : EmittedVALUInstrs) {
1234 if (MI) {
1235 if (std::optional<int> Window = WindowFor(*MI)) {
1236 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1237 return *Window - WaitStates;
1238 }
1239 }
1240
1241 ++WaitStates;
1242
1243 if (WaitStates >= MaxWindow)
1244 break;
1245 }
1246 return 0;
1247}
1248
1249int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1250 IsHazardFn IsHazardDef,
1251 int Limit) const {
1252 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1253
1254 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1255 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1256 };
1257
1258 return getWaitStatesSince(IsHazardFn, Limit);
1259}
1260
1261int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1262 int Limit) const {
1263 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1264 return isSSetReg(MI.getOpcode()) && IsHazard(MI);
1265 };
1266
1267 return getWaitStatesSince(IsHazardFn, Limit);
1268}
1269
1270//===----------------------------------------------------------------------===//
1271// No-op Hazard Detection
1272//===----------------------------------------------------------------------===//
1273
1274static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1275 MCRegister Reg) {
1276 for (MCRegUnit Unit : TRI.regunits(Reg))
1277 BV.set(static_cast<unsigned>(Unit));
1278}
1279
1282 BitVector &DefSet, BitVector &UseSet) {
1283 for (const MachineOperand &Op : Ops) {
1284 if (Op.isReg())
1285 addRegUnits(TRI, Op.isDef() ? DefSet : UseSet, Op.getReg().asMCReg());
1286 }
1287}
1288
1289void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1290 addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses);
1291}
1292
1294 return !SIInstrInfo::isSMRD(*MI);
1295}
1296
1298 return !SIInstrInfo::isVMEM(*MI);
1299}
1300
1301int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1302 // SMEM soft clause are only present on VI+, and only matter if xnack is
1303 // enabled.
1304 if (!ST.isXNACKEnabled())
1305 return 0;
1306
1307 bool IsSMRD = TII.isSMRD(*MEM);
1308
1309 resetClause();
1310
1311 // A soft-clause is any group of consecutive SMEM instructions. The
1312 // instructions in this group may return out of order and/or may be
1313 // replayed (i.e. the same instruction issued more than once).
1314 //
1315 // In order to handle these situations correctly we need to make sure that
1316 // when a clause has more than one instruction, no instruction in the clause
1317 // writes to a register that is read by another instruction in the clause
1318 // (including itself). If we encounter this situation, we need to break the
1319 // clause by inserting a non SMEM instruction.
1320
1321 for (MachineInstr *MI : EmittedInstrs) {
1322 // When we hit a non-SMEM instruction then we have passed the start of the
1323 // clause and we can stop.
1324 if (!MI)
1325 break;
1326
1328 break;
1329
1330 addClauseInst(*MI);
1331 }
1332
1333 if (ClauseDefs.none())
1334 return 0;
1335
1336 // We need to make sure not to put loads and stores in the same clause if they
1337 // use the same address. For now, just start a new clause whenever we see a
1338 // store.
1339 if (MEM->mayStore())
1340 return 1;
1341
1342 addClauseInst(*MEM);
1343
1344 // If the set of defs and uses intersect then we cannot add this instruction
1345 // to the clause, so we have a hazard.
1346 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
1347}
1348
1349int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1350 int WaitStatesNeeded = 0;
1351
1352 WaitStatesNeeded = checkSoftClauseHazards(SMRD);
1353
1354 // This SMRD hazard only affects SI.
1355 if (!ST.hasSMRDReadVALUDefHazard())
1356 return WaitStatesNeeded;
1357
1358 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1359 // SGPR was written by a VALU instruction.
1360 int SmrdSgprWaitStates = 4;
1361 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1362 return TII.isVALU(MI, /*AllowLDSDMA=*/false);
1363 };
1364 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1365 return TII.isSALU(MI);
1366 };
1367
1368 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
1369
1370 for (const MachineOperand &Use : SMRD->uses()) {
1371 if (!Use.isReg())
1372 continue;
1373 int WaitStatesNeededForUse =
1374 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
1375 SmrdSgprWaitStates);
1376 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1377
1378 // This fixes what appears to be undocumented hardware behavior in SI where
1379 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1380 // needs some number of nops in between. We don't know how many we need, but
1381 // let's use 4. This wasn't discovered before probably because the only
1382 // case when this happens is when we expand a 64-bit pointer into a full
1383 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1384 // probably never encountered in the closed-source land.
1385 if (IsBufferSMRD) {
1386 int WaitStatesNeededForUse =
1387 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
1388 IsBufferHazardDefFn,
1389 SmrdSgprWaitStates);
1390 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1391 }
1392 }
1393
1394 return WaitStatesNeeded;
1395}
1396
1397int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1398 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1399 return 0;
1400
1401 int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
1402
1403 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1404 // SGPR was written by a VALU Instruction.
1405 const int VmemSgprWaitStates = 5;
1406 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1407 return TII.isVALU(MI, /*AllowLDSDMA=*/false);
1408 };
1409 for (const MachineOperand &Use : VMEM->uses()) {
1410 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
1411 continue;
1412
1413 int WaitStatesNeededForUse =
1414 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
1415 VmemSgprWaitStates);
1416 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1417 }
1418 return WaitStatesNeeded;
1419}
1420
1421int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1422 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1423 const SIInstrInfo *TII = ST.getInstrInfo();
1424
1425 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1426 int DppVgprWaitStates = 2;
1427 int DppExecWaitStates = 5;
1428 int WaitStatesNeeded = 0;
1429 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1430 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1431 };
1432
1433 for (const MachineOperand &Use : DPP->uses()) {
1434 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
1435 continue;
1436 int WaitStatesNeededForUse =
1437 DppVgprWaitStates - getWaitStatesSinceDef(
1438 Use.getReg(),
1439 [](const MachineInstr &) { return true; },
1440 DppVgprWaitStates);
1441 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1442 }
1443
1444 WaitStatesNeeded = std::max(
1445 WaitStatesNeeded,
1446 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
1447 DppExecWaitStates));
1448
1449 return WaitStatesNeeded;
1450}
1451
1452int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1453 const SIInstrInfo *TII = ST.getInstrInfo();
1454
1455 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1456 // instruction.
1457 const int DivFMasWaitStates = 4;
1458 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1459 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1460 };
1461 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
1462 DivFMasWaitStates);
1463
1464 return DivFMasWaitStates - WaitStatesNeeded;
1465}
1466
1467int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1468 const SIInstrInfo *TII = ST.getInstrInfo();
1469 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
1470
1471 const int GetRegWaitStates = 2;
1472 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1473 return GetRegHWReg == getHWReg(TII, MI);
1474 };
1475 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
1476
1477 return GetRegWaitStates - WaitStatesNeeded;
1478}
1479
1480int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1481 const SIInstrInfo *TII = ST.getInstrInfo();
1482 unsigned HWReg = getHWReg(TII, *SetRegInstr);
1483
1484 const int SetRegWaitStates = ST.getSetRegWaitStates();
1485 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1486 return HWReg == getHWReg(TII, MI);
1487 };
1488 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
1489 return SetRegWaitStates - WaitStatesNeeded;
1490}
1491
1492int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1493 if (!MI.mayStore())
1494 return -1;
1495
1496 const SIInstrInfo *TII = ST.getInstrInfo();
1497 unsigned Opcode = MI.getOpcode();
1498 const MCInstrDesc &Desc = MI.getDesc();
1499
1500 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
1501 int VDataRCID = -1;
1502 if (VDataIdx != -1)
1503 VDataRCID = TII->getOpRegClassID(Desc.operands()[VDataIdx]);
1504
1505 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1506 // There is no hazard if the instruction does not use vector regs
1507 // (like wbinvl1)
1508 if (VDataIdx == -1)
1509 return -1;
1510 if (AMDGPU::getRegBitWidth(VDataRCID) > 64) {
1511 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1512 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1513 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1514 // Otherwise the hazard only exists if soffset is not an SGPR.
1515 if (ST.hasVDecCoExecHazard())
1516 return VDataIdx;
1517 const MachineOperand *SOffset =
1518 TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
1519 if (!SOffset || !SOffset->isReg())
1520 return VDataIdx;
1521 }
1522 }
1523
1524 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1525 // the store size is greater than 8 bytes and they have more than two bits
1526 // of their dmask set.
1527 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1528 if (TII->isMIMG(MI)) {
1529 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
1530 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1531 Desc.operands()[SRsrcIdx])) == 256);
1532 (void)SRsrcIdx;
1533 }
1534
1535 if (TII->isFLAT(MI)) {
1536 // There is no hazard if the instruction does not use vector regs
1537 if (VDataIdx == -1)
1538 return -1;
1539
1540 if (AMDGPU::getRegBitWidth(VDataRCID) > 64)
1541 return VDataIdx;
1542 }
1543
1544 return -1;
1545}
1546
1547int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1548 Register Reg) const {
1549 // Wide stores need a single wait-state bubble before a VALU that overwrites
1550 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1551 // SGPR SOFFSET.
1552 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1553
1554 auto IsHazard = [&](const MachineInstr &MI) {
1555 int DataIdx = createsVALUHazard(MI);
1556 return DataIdx >= 0 &&
1557 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
1558 };
1559
1560 return std::max(0, 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1561}
1562
1563int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1564 Register Reg) const {
1565 // The required wait-state window depends on the producer's SOFFSET shape:
1566 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1567 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1568 // store: 2 wait states.
1569 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1570 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1571 const SIInstrInfo *TII = ST.getInstrInfo();
1572
1573 int WaitStatesNeeded = 0;
1574
1575 // Scan each wait-state window separately and take the max padding needed.
1576 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1577 for (int Window = 1; Window <= 2; ++Window) {
1578 auto IsHazard = [&](const MachineInstr &MI) {
1579 int DataIdx = createsVALUHazard(MI);
1580 if (DataIdx < 0 ||
1581 !TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg))
1582 return false;
1583
1584 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1585 // with an SGPR SOFFSET, which only require a single wait state.
1586 if (Window == 1 || !TII->isBUF(MI))
1587 return true;
1588
1589 const MachineOperand *SOffset =
1590 TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
1591 return !SOffset || !SOffset->isReg();
1592 };
1593 WaitStatesNeeded = std::max(WaitStatesNeeded,
1594 Window - getWaitStatesSince(IsHazard, Window));
1595 }
1596
1597 return WaitStatesNeeded;
1598}
1599
1600int GCNHazardRecognizer::checkVALUHazardsHelper(
1601 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1602 // Helper to check for the hazard where VMEM instructions that store more
1603 // than 8 bytes can have their store data overwritten by the next
1604 // instruction.
1605 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1606
1607 if (!TRI->isVectorRegister(MRI, Def.getReg()))
1608 return 0;
1609
1610 if (ST.hasVDecCoExecHazard())
1611 return checkSOFFSETWindowVALUHazardsHelper(Def.getReg());
1612
1613 return checkUniformWindowVALUHazardsHelper(Def.getReg());
1614}
1615
1616/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1617/// pack the computed value into correct bit position of the dest register. This
1618/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1619/// dst_sel that is not aligned to the register. This function analayzes the \p
1620/// MI and \returns an operand with dst forwarding issue, or nullptr if
1621/// none exists.
1622static const MachineOperand *
1624 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1625 return nullptr;
1626
1627 const SIInstrInfo *TII = ST.getInstrInfo();
1628
1629 unsigned Opcode = MI.getOpcode();
1630
1631 // There are three different types of instructions
1632 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1633 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1634 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1635 // op_sel[3:2]
1636 // != 0
1637 if (SIInstrInfo::isSDWA(MI)) {
1638 // Type 1: SDWA with dst_sel != DWORD
1639 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
1640 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1641 return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
1642 }
1643
1644 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opcode);
1645 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel)) {
1646 // Type 2: VOP3 which write the hi bits. DST_OP_SEL is a destination
1647 // selector in the VOP3 layout; VOP3P uses this bit for source op_sel_hi.
1649 (TII->getNamedImmOperand(MI, AMDGPU::OpName::src0_modifiers) &
1651 return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
1652
1653 // Type 2b: VOP3P mix instructions with a tied destination (MIXLO/MIXHI)
1654 // write only one half of vdst and preserve the other half, so treat them
1655 // like a VOP3 destination op_sel write regardless of their source
1656 // op_sel_hi bits.
1657 if (SIInstrInfo::isVOP3P(MI) &&
1658 AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::vdst_in))
1659 return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
1660
1661 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1662 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1663 (TII->getNamedImmOperand(MI, AMDGPU::OpName::src2_modifiers) &
1665 return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
1666 }
1667
1668 // Special case: nop is required for all the opsel values for fp4 sr variant
1669 // cvt scale instructions
1670 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1671 return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
1672
1673 return nullptr;
1674}
1675
1676/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1677/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1678/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1680 const MachineOperand *Dst,
1681 const SIRegisterInfo *TRI) {
1682 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1683 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1684 // and we must account for that hazard.
1685 // We also must account for WAW hazards. In particular, WAW with dest
1686 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1687 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1688 // check for ECC. Without accounting for this hazard, the ECC will be
1689 // wrong.
1690 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1691 // complete zeroesHigh16BitsOfDest)
1692 for (auto &Operand : VALU->operands()) {
1693 if (Operand.isReg() && TRI->regsOverlap(Dst->getReg(), Operand.getReg())) {
1694 return true;
1695 }
1696 }
1697 return false;
1698}
1699
1700int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1701 int WaitStatesNeeded = 0;
1702
1703 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) {
1704 const int TransDefWaitstates = 1;
1705
1706 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1708 return false;
1709 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1710 const SIInstrInfo *TII = ST.getInstrInfo();
1711 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
1712
1713 for (const MachineOperand &Use : VALU->explicit_uses()) {
1714 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
1715 return true;
1716 }
1717
1718 return false;
1719 };
1720
1721 int WaitStatesNeededForDef =
1722 TransDefWaitstates -
1723 getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
1724 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1725 }
1726
1727 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1728 const int Shift16DefWaitstates = 1;
1729
1730 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1731 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1732 const MachineOperand *ForwardedDst =
1733 getDstSelForwardingOperand(ProducerMI, ST);
1734 if (ForwardedDst) {
1735 return consumesDstSelForwardingOperand(VALU, ForwardedDst, TRI);
1736 }
1737
1738 if (ProducerMI.isInlineAsm()) {
1739 // Assume inline asm has dst forwarding hazard
1740 for (auto &Def : ProducerMI.all_defs()) {
1741 if (consumesDstSelForwardingOperand(VALU, &Def, TRI))
1742 return true;
1743 }
1744 }
1745
1746 return false;
1747 };
1748
1749 int WaitStatesNeededForDef =
1750 Shift16DefWaitstates -
1751 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
1752 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1753 }
1754
1755 if (ST.hasVDecCoExecHazard()) {
1756 const int VALUWriteSGPRVALUReadWaitstates = 2;
1757 const int VALUWriteEXECRWLane = 4;
1758 const int VALUWriteVGPRReadlaneRead = 1;
1759
1760 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1761 const MachineRegisterInfo &MRI = MF.getRegInfo();
1763 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1764 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1765 return false;
1766 return MI.modifiesRegister(UseReg, TRI);
1767 };
1768
1769 for (const MachineOperand &Use : VALU->explicit_uses()) {
1770 if (!Use.isReg())
1771 continue;
1772
1773 UseReg = Use.getReg();
1774 if (TRI->isSGPRReg(MRI, UseReg)) {
1775 int WaitStatesNeededForDef =
1776 VALUWriteSGPRVALUReadWaitstates -
1777 getWaitStatesSince(IsVALUDefSGPRFn,
1778 VALUWriteSGPRVALUReadWaitstates);
1779 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1780 }
1781 }
1782
1783 if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
1784 UseReg = AMDGPU::VCC;
1785 int WaitStatesNeededForDef =
1786 VALUWriteSGPRVALUReadWaitstates -
1787 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
1788 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1789 }
1790
1791 switch (VALU->getOpcode()) {
1792 case AMDGPU::V_READLANE_B32:
1793 case AMDGPU::V_READFIRSTLANE_B32: {
1794 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
1795 UseReg = Src->getReg();
1796 int WaitStatesNeededForDef =
1797 VALUWriteVGPRReadlaneRead -
1798 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
1799 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1800 }
1801 [[fallthrough]];
1802 case AMDGPU::V_WRITELANE_B32: {
1803 UseReg = AMDGPU::EXEC;
1804 int WaitStatesNeededForDef =
1805 VALUWriteEXECRWLane -
1806 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
1807 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1808 break;
1809 }
1810 default:
1811 break;
1812 }
1813 }
1814
1815 // This checks for the hazard where VMEM instructions that store more than
1816 // 8 bytes can have there store data over written by the next instruction.
1817 if (!ST.has12DWordStoreHazard())
1818 return WaitStatesNeeded;
1819
1820 const MachineRegisterInfo &MRI = MF.getRegInfo();
1821
1822 for (const MachineOperand &Def : VALU->defs()) {
1823 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
1824 }
1825
1826 return WaitStatesNeeded;
1827}
1828
1829int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1830 // This checks for hazards associated with inline asm statements.
1831 // Since inline asms can contain just about anything, we use this
1832 // to call/leverage other check*Hazard routines. Note that
1833 // this function doesn't attempt to address all possible inline asm
1834 // hazards (good luck), but is a collection of what has been
1835 // problematic thus far.
1836
1837 // see checkVALUHazards()
1838 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1839 !ST.hasCvtScaleForwardingHazard())
1840 return 0;
1841
1842 const MachineRegisterInfo &MRI = MF.getRegInfo();
1843 int WaitStatesNeeded = 0;
1844
1845 for (const MachineOperand &Op :
1847 if (Op.isReg() && Op.isDef()) {
1848 if (!TRI.isVectorRegister(MRI, Op.getReg()))
1849 continue;
1850
1851 if (ST.has12DWordStoreHazard()) {
1852 WaitStatesNeeded =
1853 std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
1854 }
1855 }
1856 }
1857
1858 if (ST.hasDstSelForwardingHazard()) {
1859 const int Shift16DefWaitstates = 1;
1860
1861 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1862 const MachineOperand *Dst = getDstSelForwardingOperand(ProducerMI, ST);
1863 // Assume inline asm reads the dst
1864 if (Dst)
1865 return IA->modifiesRegister(Dst->getReg(), &TRI) ||
1866 IA->readsRegister(Dst->getReg(), &TRI);
1867
1868 if (ProducerMI.isInlineAsm()) {
1869 // If MI is inline asm, assume it has dst forwarding hazard
1870 for (auto &Def : ProducerMI.all_defs()) {
1871 if (IA->modifiesRegister(Def.getReg(), &TRI) ||
1872 IA->readsRegister(Def.getReg(), &TRI)) {
1873 return true;
1874 }
1875 }
1876 }
1877
1878 return false;
1879 };
1880
1881 int WaitStatesNeededForDef =
1882 Shift16DefWaitstates -
1883 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
1884 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1885 }
1886
1887 return WaitStatesNeeded;
1888}
1889
1890int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1891 const SIInstrInfo *TII = ST.getInstrInfo();
1892 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1893 const MachineRegisterInfo &MRI = MF.getRegInfo();
1894
1895 const MachineOperand *LaneSelectOp =
1896 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
1897
1898 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
1899 return 0;
1900
1901 Register LaneSelectReg = LaneSelectOp->getReg();
1902 auto IsHazardFn = [TII](const MachineInstr &MI) {
1903 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1904 };
1905
1906 const int RWLaneWaitStates = 4;
1907 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1908 RWLaneWaitStates);
1909 return RWLaneWaitStates - WaitStatesSince;
1910}
1911
1912int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1913 if (!ST.hasRFEHazards())
1914 return 0;
1915
1916 const SIInstrInfo *TII = ST.getInstrInfo();
1917
1918 const int RFEWaitStates = 1;
1919
1920 auto IsHazardFn = [TII](const MachineInstr &MI) {
1921 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1922 };
1923 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1924 return RFEWaitStates - WaitStatesNeeded;
1925}
1926
1927int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1928 const SIInstrInfo *TII = ST.getInstrInfo();
1929 const int ReadM0WaitStates = 1;
1930 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1931 return ReadM0WaitStates -
1932 getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1933}
1934
1935void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1937 int WaitStatesNeeded, bool IsHoisting) {
1938 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1939 for (int I = 0; I < WaitStatesNeeded; ++I)
1940 BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::V_NOP_e32));
1941}
1942
1943void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1944 fixVMEMtoScalarWriteHazards(MI);
1945 fixVcmpxPermlaneHazards(MI);
1946 fixSMEMtoVectorWriteHazards(MI);
1947 fixVcmpxExecWARHazard(MI);
1948 fixLdsBranchVmemWARHazard(MI);
1949 if (ST.hasLdsDirect()) {
1950 fixLdsDirectVALUHazard(MI);
1951 fixLdsDirectVMEMHazard(MI);
1952 }
1953 fixVALUPartialForwardingHazard(MI);
1954 fixVALUTransUseHazard(MI);
1955 fixVALUTransCoexecutionHazards(MI);
1956 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1957 fixWMMACoexecutionHazards(MI);
1958 fixShift64HighRegBug(MI);
1959 fixVALUMaskWriteHazard(MI);
1960 fixRequiredExportPriority(MI);
1961 if (ST.hasVPermPk16Hazard())
1962 fixVPermPk16Hazard(MI);
1963 if (ST.requiresWaitIdleBeforeGetReg())
1964 fixGetRegWaitIdle(MI);
1965 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1966 fixDsAtomicAsyncBarrierArriveB64(MI);
1967 if (ST.hasScratchBaseForwardingHazard())
1968 fixScratchBaseForwardingHazard(MI);
1969 if (ST.setRegModeNeedsVNOPs())
1970 fixSetRegMode(MI);
1971 if (ST.hasNeedsTDMDrain())
1972 fixTDM(MI);
1973 if (ST.hasTransF16ToPartialVGPRWriteHazard())
1974 fixTransF16ToPartialVGPRWriteHazard(MI);
1975}
1976
1978 const MachineInstr &MI) {
1979 return (TII.isVOPC(MI) ||
1980 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1981 MI.modifiesRegister(AMDGPU::EXEC, &TRI);
1982}
1983
1984bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1985 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1986 return false;
1987
1988 const SIInstrInfo *TII = ST.getInstrInfo();
1989 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1990 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1991 return isVCmpXWritesExec(*TII, *TRI, MI);
1992 };
1993
1994 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1995 unsigned Opc = MI.getOpcode();
1996 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
1997 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
1998 Opc != AMDGPU::V_NOP_sdwa;
1999 };
2000
2001 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2002 std::numeric_limits<int>::max())
2003 return false;
2004
2005 // V_NOP will be discarded by SQ.
2006 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
2007 // which is always a VGPR and available.
2008 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
2009 Register Reg = Src0->getReg();
2010 bool IsUndef = Src0->isUndef();
2011 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2012 TII->get(AMDGPU::V_MOV_B32_e32))
2015
2016 return true;
2017}
2018
2019bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
2020 if (!ST.hasVMEMtoScalarWriteHazard())
2021 return false;
2022 assert(!ST.hasExtendedWaitCounts());
2023
2025 return false;
2026
2027 if (MI->getNumDefs() == 0)
2028 return false;
2029
2030 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2031
2032 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
2034 return false;
2035
2036 for (const MachineOperand &Def : MI->defs()) {
2037 const MachineOperand *Op =
2038 I.findRegisterUseOperand(Def.getReg(), TRI, false);
2039 if (!Op)
2040 continue;
2041 return true;
2042 }
2043 return false;
2044 };
2045
2046 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2047 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
2048 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
2049 !MI.getOperand(0).getImm()) ||
2050 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2052 static_cast<unsigned>(MI.getOperand(0).getImm())) == 0);
2053 };
2054
2055 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2056 std::numeric_limits<int>::max())
2057 return false;
2058
2059 const SIInstrInfo *TII = ST.getInstrInfo();
2060 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2061 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
2063 return true;
2064}
2065
2066bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
2067 if (!ST.hasSMEMtoVectorWriteHazard())
2068 return false;
2069 assert(!ST.hasExtendedWaitCounts());
2070
2071 if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false))
2072 return false;
2073
2074 AMDGPU::OpName SDSTName;
2075 switch (MI->getOpcode()) {
2076 case AMDGPU::V_READLANE_B32:
2077 case AMDGPU::V_READFIRSTLANE_B32:
2078 SDSTName = AMDGPU::OpName::vdst;
2079 break;
2080 default:
2081 SDSTName = AMDGPU::OpName::sdst;
2082 break;
2083 }
2084
2085 const SIInstrInfo *TII = ST.getInstrInfo();
2086 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2087 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
2088 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
2089 if (!SDST) {
2090 for (const auto &MO : MI->implicit_operands()) {
2091 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) {
2092 SDST = &MO;
2093 break;
2094 }
2095 }
2096 }
2097
2098 if (!SDST)
2099 return false;
2100
2101 const Register SDSTReg = SDST->getReg();
2102 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
2103 return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
2104 };
2105
2106 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
2107 if (TII->isSALU(MI)) {
2108 switch (MI.getOpcode()) {
2109 case AMDGPU::S_SETVSKIP:
2110 case AMDGPU::S_VERSION:
2111 case AMDGPU::S_WAITCNT_VSCNT:
2112 case AMDGPU::S_WAITCNT_VMCNT:
2113 case AMDGPU::S_WAITCNT_EXPCNT:
2114 // These instructions cannot not mitigate the hazard.
2115 return false;
2116 case AMDGPU::S_WAITCNT_LGKMCNT:
2117 // Reducing lgkmcnt count to 0 always mitigates the hazard.
2118 return (MI.getOperand(1).getImm() == 0) &&
2119 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
2120 case AMDGPU::S_WAITCNT: {
2121 const int64_t Imm = MI.getOperand(0).getImm();
2122 AMDGPU::Waitcnt Decoded =
2123 AMDGPU::decodeWaitcnt(IV, static_cast<unsigned>(Imm));
2124 // DsCnt corresponds to LGKMCnt here.
2125 return Decoded.get(AMDGPU::DS_CNT) == 0;
2126 }
2127 default:
2128 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
2129 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
2130 "unexpected wait count instruction");
2131 // SOPP instructions cannot mitigate the hazard.
2132 if (TII->isSOPP(MI))
2133 return false;
2134 // At this point the SALU can be assumed to mitigate the hazard
2135 // because either:
2136 // (a) it is independent of the at risk SMEM (breaking chain),
2137 // or
2138 // (b) it is dependent on the SMEM, in which case an appropriate
2139 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
2140 // SMEM instruction.
2141 return true;
2142 }
2143 }
2144 return false;
2145 };
2146
2147 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2148 std::numeric_limits<int>::max())
2149 return false;
2150
2151 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2152 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
2153 .addImm(0);
2154 return true;
2155}
2156
2157bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
2158 if (!ST.hasVcmpxExecWARHazard())
2159 return false;
2160 assert(!ST.hasExtendedWaitCounts());
2161
2162 if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false))
2163 return false;
2164
2165 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2166 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
2167 return false;
2168
2169 auto IsHazardFn = [TRI](const MachineInstr &I) {
2170 if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
2171 return false;
2172 return I.readsRegister(AMDGPU::EXEC, TRI);
2173 };
2174
2175 const SIInstrInfo *TII = ST.getInstrInfo();
2176 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
2177 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false)) {
2178 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
2179 return true;
2180 for (auto MO : MI.implicit_operands())
2181 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg())))
2182 return true;
2183 }
2184 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2186 static_cast<unsigned>(MI.getOperand(0).getImm())) == 0)
2187 return true;
2188 return false;
2189 };
2190
2191 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2192 std::numeric_limits<int>::max())
2193 return false;
2194
2195 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2196 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
2198 return true;
2199}
2200
2202 const GCNSubtarget &ST) {
2203 if (!ST.hasLdsBranchVmemWARHazard())
2204 return false;
2205
2206 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2207 // instructions need to appear in the same function.
2208 bool HasLds = false;
2209 bool HasVmem = false;
2210 for (auto &MBB : MF) {
2211 for (auto &MI : MBB) {
2213 HasVmem |= SIInstrInfo::isVMEM(MI);
2214 if (HasLds && HasVmem)
2215 return true;
2216 }
2217 }
2218 return false;
2219}
2220
2222 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2223 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
2224 !I.getOperand(1).getImm();
2225}
2226
2227bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2228 if (!RunLdsBranchVmemWARHazardFixup)
2229 return false;
2230
2231 assert(ST.hasLdsBranchVmemWARHazard());
2232 assert(!ST.hasExtendedWaitCounts());
2233
2234 auto IsHazardInst = [](const MachineInstr &MI) {
2236 return 1;
2238 return 2;
2239 return 0;
2240 };
2241
2242 auto InstType = IsHazardInst(*MI);
2243 if (!InstType)
2244 return false;
2245
2246 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2247 return IsHazardInst(I) || isStoreCountWaitZero(I);
2248 };
2249
2250 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2251 if (!I.isBranch())
2252 return false;
2253
2254 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2255 auto InstType2 = IsHazardInst(I);
2256 return InstType2 && InstType != InstType2;
2257 };
2258
2259 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2260 auto InstType2 = IsHazardInst(I);
2261 if (InstType == InstType2)
2262 return true;
2263
2264 return isStoreCountWaitZero(I);
2265 };
2266
2267 return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
2268 std::numeric_limits<int>::max();
2269 };
2270
2271 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2272 std::numeric_limits<int>::max())
2273 return false;
2274
2275 const SIInstrInfo *TII = ST.getInstrInfo();
2276 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2277 TII->get(AMDGPU::S_WAITCNT_VSCNT))
2278 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
2279 .addImm(0);
2280
2281 return true;
2282}
2283
2284bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2286 return false;
2287
2288 const int NoHazardWaitStates = 15;
2289 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
2290 const Register VDSTReg = VDST->getReg();
2291
2292 bool VisitedTrans = false;
2293 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2294 if (!SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
2295 return false;
2296 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
2297 // Cover both WAR and WAW
2298 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
2299 };
2300 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2301 if (WaitStates >= NoHazardWaitStates)
2302 return true;
2303 // Instructions which cause va_vdst==0 expire hazard
2306 };
2307 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2308 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2309 };
2310
2311 DenseSet<const MachineBasicBlock *> Visited;
2312 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
2313 std::next(MI->getReverseIterator()), 0,
2314 IsExpiredFn, Visited, GetWaitStatesFn);
2315
2316 // Transcendentals can execute in parallel to other VALUs.
2317 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2318 if (VisitedTrans)
2319 Count = 0;
2320
2321 MachineOperand *WaitVdstOp =
2322 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst);
2323 WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates));
2324
2325 return true;
2326}
2327
2328bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2330 return false;
2331
2332 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
2333 const Register VDSTReg = VDST->getReg();
2334
2335 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2337 return false;
2338 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
2339 };
2340 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2341 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2342 // according to the type of VMEM instruction.
2343 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2344 return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) ||
2346 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
2347 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2349 static_cast<unsigned>(I.getOperand(0).getImm())) == 0) ||
2350 (LdsdirCanWait && SIInstrInfo::isLDSDIR(I) &&
2351 !TII.getNamedOperand(I, AMDGPU::OpName::waitvsrc)->getImm());
2352 };
2353
2354 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2355 std::numeric_limits<int>::max())
2356 return false;
2357
2358 if (LdsdirCanWait) {
2359 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0);
2360 } else {
2361 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2362 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2364 }
2365
2366 return true;
2367}
2368
2369bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2370 if (!ST.hasVALUPartialForwardingHazard())
2371 return false;
2372 assert(!ST.hasExtendedWaitCounts());
2373
2374 if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false))
2375 return false;
2376
2377 SmallSetVector<Register, 4> SrcVGPRs;
2378
2379 for (const MachineOperand &Use : MI->explicit_uses()) {
2380 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
2381 SrcVGPRs.insert(Use.getReg());
2382 }
2383
2384 // Only applies with >= 2 unique VGPR sources
2385 if (SrcVGPRs.size() <= 1)
2386 return false;
2387
2388 // Look for the following pattern:
2389 // Va <- VALU [PreExecPos]
2390 // intv1
2391 // Exec <- SALU [ExecPos]
2392 // intv2
2393 // Vb <- VALU [PostExecPos]
2394 // intv3
2395 // MI Va, Vb (WaitState = 0)
2396 //
2397 // Where:
2398 // intv1 + intv2 <= 2 VALUs
2399 // intv3 <= 4 VALUs
2400 //
2401 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2402
2403 const int Intv1plus2MaxVALUs = 2;
2404 const int Intv3MaxVALUs = 4;
2405 const int IntvMaxVALUs = 6;
2406 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2407
2408 struct StateType {
2409 SmallDenseMap<Register, int, 4> DefPos;
2410 int ExecPos = std::numeric_limits<int>::max();
2411 int VALUs = 0;
2412
2413 static unsigned getHashValue(const StateType &State) {
2414 hash_code H = hash_combine(State.ExecPos, State.VALUs);
2415 for (const auto &[Reg, Pos] : State.DefPos)
2416 H = hash_combine(H, Reg, Pos);
2417 return static_cast<unsigned>(H);
2418 }
2419 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2420 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2421 LHS.VALUs == RHS.VALUs;
2422 }
2423 };
2424
2425 StateType State;
2426
2427 // This overloads expiry testing with all the hazard detection
2428 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2429 // Too many VALU states have passed
2430 if (State.VALUs > NoHazardVALUWaitStates)
2431 return HazardExpired;
2432
2433 // Instructions which cause va_vdst==0 expire hazard
2436 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2438 static_cast<unsigned>(I.getOperand(0).getImm())) == 0))
2439 return HazardExpired;
2440
2441 // Track registers writes
2442 bool Changed = false;
2443 if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/false)) {
2444 for (Register Src : SrcVGPRs) {
2445 if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
2446 State.DefPos[Src] = State.VALUs;
2447 Changed = true;
2448 }
2449 }
2450 } else if (SIInstrInfo::isSALU(I)) {
2451 if (State.ExecPos == std::numeric_limits<int>::max()) {
2452 if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
2453 State.ExecPos = State.VALUs;
2454 Changed = true;
2455 }
2456 }
2457 }
2458
2459 // Early expiration: too many VALUs in intv3
2460 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2461 return HazardExpired;
2462
2463 // Only evaluate state if something changed
2464 if (!Changed)
2465 return NoHazardFound;
2466
2467 // Determine positions of VALUs pre/post exec change
2468 if (State.ExecPos == std::numeric_limits<int>::max())
2469 return NoHazardFound;
2470
2471 int PreExecPos = std::numeric_limits<int>::max();
2472 int PostExecPos = std::numeric_limits<int>::max();
2473
2474 for (auto Entry : State.DefPos) {
2475 int DefVALUs = Entry.second;
2476 if (DefVALUs != std::numeric_limits<int>::max()) {
2477 if (DefVALUs >= State.ExecPos)
2478 PreExecPos = std::min(PreExecPos, DefVALUs);
2479 else
2480 PostExecPos = std::min(PostExecPos, DefVALUs);
2481 }
2482 }
2483
2484 // Need a VALUs post exec change
2485 if (PostExecPos == std::numeric_limits<int>::max())
2486 return NoHazardFound;
2487
2488 // Too many VALUs in intv3?
2489 int Intv3VALUs = PostExecPos;
2490 if (Intv3VALUs > Intv3MaxVALUs)
2491 return HazardExpired;
2492
2493 // Too many VALUs in intv2?
2494 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2495 if (Intv2VALUs > Intv1plus2MaxVALUs)
2496 return HazardExpired;
2497
2498 // Need a VALUs pre exec change
2499 if (PreExecPos == std::numeric_limits<int>::max())
2500 return NoHazardFound;
2501
2502 // Too many VALUs in intv1?
2503 int Intv1VALUs = PreExecPos - State.ExecPos;
2504 if (Intv1VALUs > Intv1plus2MaxVALUs)
2505 return HazardExpired;
2506
2507 // Too many VALUs in intv1 + intv2
2508 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2509 return HazardExpired;
2510
2511 return HazardFound;
2512 };
2513 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2514 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
2515 State.VALUs += 1;
2516 };
2517
2518 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
2519 std::next(MI->getReverseIterator())))
2520 return false;
2521
2522 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2523 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2525
2526 return true;
2527}
2528
2529bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2530 if (!ST.hasVALUTransUseHazard())
2531 return false;
2532 assert(!ST.hasExtendedWaitCounts());
2533
2534 if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false))
2535 return false;
2536
2537 SmallSet<Register, 4> SrcVGPRs;
2538
2539 for (const MachineOperand &Use : MI->explicit_uses()) {
2540 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
2541 SrcVGPRs.insert(Use.getReg());
2542 }
2543
2544 // Look for the following pattern:
2545 // Va <- TRANS VALU
2546 // intv
2547 // MI Va (WaitState = 0)
2548 //
2549 // Where:
2550 // intv <= 5 VALUs / 1 TRANS
2551 //
2552 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2553
2554 const int IntvMaxVALUs = 5;
2555 const int IntvMaxTRANS = 1;
2556
2557 struct StateType {
2558 int VALUs = 0;
2559 int TRANS = 0;
2560
2561 static unsigned getHashValue(const StateType &State) {
2562 return static_cast<unsigned>(hash_combine(State.VALUs, State.TRANS));
2563 }
2564 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2565 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2566 }
2567 };
2568
2569 StateType State;
2570
2571 // This overloads expiry testing with all the hazard detection
2572 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2573 // Too many VALU states have passed
2574 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2575 return HazardExpired;
2576
2577 // Instructions which cause va_vdst==0 expire hazard
2580 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2582 static_cast<unsigned>(I.getOperand(0).getImm())) == 0))
2583 return HazardExpired;
2584
2585 // Track registers writes
2586 if (SIInstrInfo::isTRANS(I)) {
2587 for (Register Src : SrcVGPRs) {
2588 if (I.modifiesRegister(Src, &TRI)) {
2589 return HazardFound;
2590 }
2591 }
2592 }
2593
2594 return NoHazardFound;
2595 };
2596 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2597 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
2598 State.VALUs += 1;
2600 State.TRANS += 1;
2601 };
2602
2603 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
2604 std::next(MI->getReverseIterator())))
2605 return false;
2606
2607 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2608 // avoided.
2609 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2610 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2612
2613 return true;
2614}
2615
2616bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2617 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2618 !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
2620 return false;
2621
2622 const SIInstrInfo *TII = ST.getInstrInfo();
2623 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2624
2625 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2626 if (!SIInstrInfo::isTRANS(I))
2627 return false;
2628
2629 // RAW: Trans(I) writes, VALU(MI) reads.
2630 Register TransDef = TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
2631 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2632 if (ValuUse.isReg() && TRI->regsOverlap(TransDef, ValuUse.getReg()))
2633 return true;
2634 }
2635
2636 // WAR: Trans(I) reads, VALU(MI) writes. Use modifiesRegister for
2637 // multi-dest VALUs.
2638 for (const MachineOperand &TransUse : I.explicit_uses()) {
2639 if (TransUse.isReg() && MI->modifiesRegister(TransUse.getReg(), TRI))
2640 return true;
2641 }
2642
2643 return false;
2644 };
2645
2646 auto IsExpiredFn = [](const MachineInstr &I, int) {
2647 return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
2648 };
2649
2650 const int HasVALU = std::numeric_limits<int>::max();
2651 if (::getWaitStatesSince(IsTransHazardFn, MI, IsExpiredFn) == HasVALU)
2652 return false;
2653
2654 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
2655 return true;
2656}
2657
2658bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2660 return false;
2661
2662 const SIInstrInfo *TII = ST.getInstrInfo();
2663 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2664
2665 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2667 return false;
2668
2669 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2670 // with the dest(matrix D) of the previous wmma.
2671 const Register CurSrc0Reg =
2672 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg();
2673 const Register CurSrc1Reg =
2674 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
2675
2676 const Register PrevDstReg =
2677 TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
2678
2679 if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
2680 TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
2681 return true;
2682 }
2683
2684 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2685 // but Index can't overlap with PrevDstReg.
2686 if (AMDGPU::isGFX12Plus(ST)) {
2687 if (SIInstrInfo::isSWMMAC(*MI)) {
2688 const Register CurIndex =
2689 TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg();
2690 if (TRI->regsOverlap(PrevDstReg, CurIndex))
2691 return true;
2692 }
2693 return false;
2694 }
2695
2696 return false;
2697 };
2698
2699 auto IsExpiredFn = [](const MachineInstr &I, int) {
2700 return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
2701 };
2702
2703 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2704 std::numeric_limits<int>::max())
2705 return false;
2706
2707 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
2708
2709 return true;
2710}
2711
2713 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
2715}
2716
2717// Classify XDL WMMA instructions into co-execution hazard categories
2718// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2719//
2720// Category 0: WMMA with Latency 8
2721// WMMA_*F16, WMMA_*BF16
2722// WMMA_*_16X16X128_{FP8,BF8}
2723// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2724//
2725// Category 1: WMMA Latency 16
2726// WMMA_IU8
2727//
2728// Category 2: SWMMAC with Latency 8
2729// SWMMAC_*F16, SWMMAC_*BF16,
2730// SWMMAC_*FP8FP8
2731// SWMMAC_*BF8FP8
2732// SWMMAC_*FP8BF8
2733// SWMMAC_*BF8BF8
2734//
2735// Category 3: SWMMAC with Latency 16
2736// SWMMAC_IU8
2737//
2738// Category 4: 16 Pass GFX1251 WMMA with latency 16
2739// V_WMMA_*_16X16X32_{F16,BF16}
2740// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2741// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2742// V_SWMMAC_*_16X16X64_{F16,BF16}
2743// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2744//
2745// Category 5: 32 Pass GFX1251 WMMA with latency 32
2746// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2747// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2748// V_WMMA_F32_32X16X128_F4
2749// V_WMMA_I32_16X16X64_IU8
2750// V_WMMA_I32_16X16X64_IU8
2751//
2752// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2753// WMMA_*_16X16X64_{FP8,BF8}
2754// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2756 const SIInstrInfo *TII,
2757 const TargetSchedModel &SchedModel,
2758 const GCNSubtarget &ST) {
2759 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2760 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2761 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2762 unsigned Category = 0;
2763
2764 unsigned Latency = SchedModel.computeInstrLatency(&MI);
2765 switch (Latency) {
2766 case 4:
2767 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2768 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2769 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2770 Category = 6;
2771 break;
2772 case 8:
2773 Category = IsSWMMAC ? 2 : 0;
2774 break;
2775 case 16:
2776 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2777 break;
2778 case 32:
2779 assert(IsLowestRateWMMA && "latency 32 is not expected");
2780 Category = 5;
2781 break;
2782 default:
2783 llvm_unreachable("unexpected xdl wmma latency");
2784 } // end switch.
2785
2786 return Category;
2787}
2788
2789int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2790 if (!ST.hasWMMACoexecutionHazards())
2791 return 0;
2792
2793 const SIInstrInfo *TII = ST.getInstrInfo();
2794 if (!TII->isXDLWMMA(*MI) && !isCoexecutableVALUInst(*MI))
2795 return 0;
2796
2797 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2798 // be in between the first WMMA and the second instruction to cover the hazard
2799 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2800 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2801 // numbers, which depends on the category of the first WMMA.
2802 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2803 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2804 const bool IsWMMA = TII->isXDLWMMA(*MI);
2805 const bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2806 ArrayRef<int> RequiredWaitStates =
2807 IsWMMA ? ArrayRef<int>(WMMAWaitStates) : ArrayRef<int>(VALUWaitStates);
2808 const int MaxWaitStates =
2809 IsWMMA ? (IsLowestRateWMMA ? 17 : 9) : (IsLowestRateWMMA ? 16 : 8);
2810
2811 auto WindowFor = [=](const MachineInstr &I) -> std::optional<int> {
2812 if (!TII->isXDLWMMA(I))
2813 return std::nullopt;
2814 if (IsWMMA ? !hasWMMAToWMMARegOverlap(I, *MI)
2815 : !hasWMMAToVALURegOverlap(I, *MI))
2816 return std::nullopt;
2817 return RequiredWaitStates[getWMMAHazardInstInCategory(I, TII, TSchedModel,
2818 ST)];
2819 };
2820
2821 return getMaxVALUWindowDeficit(MaxWaitStates, WindowFor);
2822}
2823
2824bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2825 const MachineInstr &WMMA, const MachineInstr &MI) const {
2826 Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
2827 Register A1 = TII.getNamedOperand(MI, AMDGPU::OpName::src0)->getReg();
2828 Register B1 = TII.getNamedOperand(MI, AMDGPU::OpName::src1)->getReg();
2829
2830 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2831 if (TRI.regsOverlap(D0, A1) || TRI.regsOverlap(D0, B1))
2832 return true;
2833
2835 Register Idx1 = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
2836 if (TRI.regsOverlap(D0, Idx1))
2837 return true;
2838 }
2839 return false;
2840}
2841
2842bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2843 const MachineInstr &WMMA, const MachineInstr &MI) const {
2844 // WMMA writes, VALU reads.
2845 Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
2846 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2847 if (ValuUse.isReg() && TRI.regsOverlap(D0, ValuUse.getReg()))
2848 return true;
2849 }
2850
2851 // WMMA reads or writes, VALU writes.
2852 Register A0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src0)->getReg();
2853 Register B0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src1)->getReg();
2854 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2855
2856 const MachineOperand *Src2 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src2);
2857 if (Src2->isReg())
2858 WMMARegs.push_back(Src2->getReg());
2859
2860 for (const MachineOperand &ValuDef : MI.defs()) {
2861 Register VDstReg = ValuDef.getReg();
2862 for (Register WMMAReg : WMMARegs) {
2863 if (TRI.regsOverlap(VDstReg, WMMAReg))
2864 return true;
2865 }
2866 }
2867 return false;
2868}
2869
2870bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2871 const MachineInstr &MI) const {
2872 // I is the potential WMMA hazard source, MI is the instruction being checked
2873 // for hazard.
2874 if (!TII.isXDLWMMA(I))
2875 return false;
2876
2877 // Dispatch based on MI type
2878 if (TII.isXDLWMMA(MI))
2879 return hasWMMAToWMMARegOverlap(I, MI);
2881 return hasWMMAToVALURegOverlap(I, MI);
2882
2883 return false;
2884}
2885
2886bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2887 bool IncludeSubloops) {
2888 // Scan loop for any WMMA that hazards MI.
2889 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2890 for (MachineBasicBlock *MBB : L->getBlocks()) {
2891 if (!IncludeSubloops && MLI->getLoopFor(MBB) != L)
2892 continue;
2893 for (MachineInstr &I : *MBB) {
2894 if (&I == MI)
2895 continue;
2896 if (isCoexecutionHazardFor(I, *MI))
2897 return true;
2898 }
2899 }
2900 return false;
2901}
2902
2903bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2904 int WaitStatesNeeded) {
2905 if (!MLI)
2906 return false;
2907
2908 MachineLoop *L = MLI->getLoopFor(MI->getParent());
2909 if (!L) {
2910 ++NumWMMAHoistingBailed;
2911 return false;
2912 }
2913
2914 // If innermost loop has WMMA hazard, we can't hoist at all
2915 if (hasWMMAHazardInLoop(L, MI)) {
2916 ++NumWMMAHoistingBailed;
2917 return false;
2918 }
2919
2920 // Find outermost loop with no internal hazard
2921 MachineLoop *TargetLoop = L;
2922 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2923 if (hasWMMAHazardInLoop(Parent, MI, false))
2924 break; // Parent has hazard in its own blocks, stop here
2925 TargetLoop = Parent; // Safe to hoist further out
2926 }
2927
2928 // Need valid preheader to insert V_NOPs
2929 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2930 if (!Preheader) {
2931 ++NumWMMAHoistingBailed;
2932 return false;
2933 }
2934
2935 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2936 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2937 << "\n");
2938
2939 emitVNops(*Preheader, Preheader->getFirstTerminator(), WaitStatesNeeded,
2940 /*IsHoisting=*/true);
2941 NumWMMANopsHoisted += WaitStatesNeeded;
2942 return true;
2943}
2944
2945bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2946 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2947 if (WaitStatesNeeded <= 0)
2948 return false;
2949
2950 if (ST.getCLOpts().wmma_vnop_hoisting &&
2951 tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2952 return true;
2953
2954 emitVNops(*MI->getParent(), MI->getIterator(), WaitStatesNeeded);
2955 return true;
2956}
2957
2958bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2959 if (!ST.hasShift64HighRegBug())
2960 return false;
2961 assert(!ST.hasExtendedWaitCounts());
2962
2963 switch (MI->getOpcode()) {
2964 default:
2965 return false;
2966 case AMDGPU::V_LSHLREV_B64_e64:
2967 case AMDGPU::V_LSHRREV_B64_e64:
2968 case AMDGPU::V_ASHRREV_I64_e64:
2969 break;
2970 }
2971
2972 MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0);
2973 if (!Amt->isReg())
2974 return false;
2975
2976 Register AmtReg = Amt->getReg();
2977 const MachineRegisterInfo &MRI = MF.getRegInfo();
2978 // Check if this is a last VGPR in the allocation block.
2979 if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2980 return false;
2981
2982 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1))
2983 return false;
2984
2985 assert(ST.needsAlignedVGPRs());
2986 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2987
2988 const DebugLoc &DL = MI->getDebugLoc();
2989 MachineBasicBlock *MBB = MI->getParent();
2990 MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1);
2991
2992 // In:
2993 //
2994 // Dst = shiftrev64 Amt, Src1
2995 //
2996 // if Dst!=Src1 then avoid the bug with:
2997 //
2998 // Dst.sub0 = Amt
2999 // Dst = shift64 Dst.sub0, Src1
3000
3001 Register DstReg = MI->getOperand(0).getReg();
3002 if (!Src1->isReg() || Src1->getReg() != DstReg) {
3003 Register DstLo = TRI.getSubReg(DstReg, AMDGPU::sub0);
3004 runOnInstruction(
3005 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstLo).add(*Amt));
3006 Amt->setReg(DstLo);
3007 Amt->setIsKill(true);
3008 return true;
3009 }
3010
3011 bool Overlapped = MI->modifiesRegister(AmtReg, &TRI);
3012 Register NewReg;
3013 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
3014 : AMDGPU::VGPR_32RegClass) {
3015 if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) {
3016 NewReg = Reg;
3017 break;
3018 }
3019 }
3020
3021 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1)
3022 : NewReg;
3023 Register NewAmtLo;
3024
3025 if (Overlapped)
3026 NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0);
3027
3028 // Insert a full wait count because found register might be pending a wait.
3029 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT))
3030 .addImm(0);
3031
3032 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
3033 if (Overlapped)
3034 runOnInstruction(
3035 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo)
3036 .addDef(AmtReg - 1)
3037 .addReg(AmtReg - 1, RegState::Undef)
3038 .addReg(NewAmtLo, RegState::Undef));
3039 runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt)
3040 .addDef(AmtReg)
3041 .addReg(AmtReg, RegState::Undef)
3042 .addReg(NewAmt, RegState::Undef));
3043
3044 // Instructions emitted after the current instruction will be processed by the
3045 // parent loop of the hazard recognizer in a natural way.
3046 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
3047 AmtReg)
3048 .addDef(NewAmt)
3049 .addReg(NewAmt)
3050 .addReg(AmtReg);
3051 if (Overlapped)
3052 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
3053 AmtReg - 1)
3054 .addDef(NewAmtLo)
3055 .addReg(NewAmtLo)
3056 .addReg(AmtReg - 1);
3057
3058 // Re-running hazard recognizer on the modified instruction is not necessary,
3059 // inserted V_SWAP_B32 has already both read and write new registers so
3060 // hazards related to these register has already been handled.
3061 Amt->setReg(NewAmt);
3062 Amt->setIsKill(false);
3063 // We do not update liveness, so verifier may see it as undef.
3064 Amt->setIsUndef();
3065 if (Overlapped) {
3066 MI->getOperand(0).setReg(NewReg);
3067 Src1->setReg(NewReg);
3068 Src1->setIsKill(false);
3069 Src1->setIsUndef();
3070 }
3071
3072 return true;
3073}
3074
3075int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
3076 int NSAtoVMEMWaitStates = 1;
3077
3078 if (!ST.hasNSAtoVMEMBug())
3079 return 0;
3080
3082 return 0;
3083
3084 const SIInstrInfo *TII = ST.getInstrInfo();
3085 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
3086 if (!Offset || (Offset->getImm() & 6) == 0)
3087 return 0;
3088
3089 auto IsHazardFn = [TII](const MachineInstr &I) {
3090 if (!SIInstrInfo::isMIMG(I))
3091 return false;
3092 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
3093 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
3094 TII->getInstSizeInBytes(I) >= 16;
3095 };
3096
3097 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
3098}
3099
3100int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
3101 MachineInstr *MI) const {
3102 int FPAtomicToDenormModeWaitStates = 3;
3103
3104 if (!ST.hasFPAtomicToDenormModeHazard())
3105 return 0;
3106 assert(!ST.hasExtendedWaitCounts());
3107
3108 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
3109 return 0;
3110
3111 auto IsHazardFn = [](const MachineInstr &I) {
3112 if (!SIInstrInfo::isVMEM(I))
3113 return false;
3114 return SIInstrInfo::isFPAtomic(I);
3115 };
3116
3117 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
3118 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
3119 return true;
3120
3121 return SIInstrInfo::isWaitcnt(MI.getOpcode());
3122 };
3123
3124 return FPAtomicToDenormModeWaitStates -
3125 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
3126}
3127
3128int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
3130
3131 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
3132}
3133
3134int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
3135 // Early exit if no padding is requested.
3136 if (MFMAPaddingRatio == 0)
3137 return 0;
3138
3139 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3140 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
3141 return 0;
3142
3143 int NeighborMFMALatency = 0;
3144 auto IsNeighboringMFMA = [&NeighborMFMALatency,
3145 this](const MachineInstr &MI) {
3146 if (!SIInstrInfo::isMFMA(MI))
3147 return false;
3148
3149 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
3150 return true;
3151 };
3152
3153 const int MaxMFMAPipelineWaitStates = 16;
3154 int WaitStatesSinceNeighborMFMA =
3155 getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
3156
3157 int NeighborMFMAPaddingNeeded =
3158 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
3159 WaitStatesSinceNeighborMFMA;
3160
3161 return std::max(0, NeighborMFMAPaddingNeeded);
3162}
3163
3164int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
3165 int WaitStatesNeeded = 0;
3166 unsigned Opc = MI->getOpcode();
3167
3168 auto IsVALUFn = [](const MachineInstr &MI) {
3169 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) || MI.isInlineAsm();
3170 };
3171
3172 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3173 const int LegacyVALUWritesVGPRWaitStates = 2;
3174 const int VALUWritesExecWaitStates = 4;
3175 const int MaxWaitStates = 4;
3176
3177 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3178 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
3179 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3180
3181 if (WaitStatesNeeded < MaxWaitStates) {
3182 for (const MachineOperand &Use : MI->explicit_uses()) {
3183 const int MaxWaitStates = 2;
3184
3185 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
3186 continue;
3187
3188 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3189 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
3190 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3191
3192 if (WaitStatesNeeded == MaxWaitStates)
3193 break;
3194 }
3195 }
3196 }
3197
3198 for (const MachineOperand &Op : MI->explicit_operands()) {
3199 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
3200 continue;
3201
3202 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3203 continue;
3204
3205 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3206 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3207 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3208 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3209 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3210 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3211 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3212 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3213 const int MaxWaitStates = 18;
3214 Register Reg = Op.getReg();
3215 unsigned HazardDefLatency = 0;
3216
3217 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3218 this](const MachineInstr &MI) {
3219 if (!SIInstrInfo::isMFMA(MI))
3220 return false;
3221 Register DstReg = MI.getOperand(0).getReg();
3222 if (DstReg == Reg)
3223 return false;
3224 HazardDefLatency =
3225 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
3226 return TRI.regsOverlap(DstReg, Reg);
3227 };
3228
3229 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
3230 MaxWaitStates);
3231 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3232 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
3233 int OpNo = Op.getOperandNo();
3234 if (OpNo == SrcCIdx) {
3235 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3236 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3237 switch (HazardDefLatency) {
3238 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3239 break;
3240 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3241 break;
3242 case 16: [[fallthrough]];
3243 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3244 break;
3245 }
3246 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3247 switch (HazardDefLatency) {
3248 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3249 break;
3250 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3251 break;
3252 case 16: [[fallthrough]];
3253 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3254 break;
3255 }
3256 }
3257
3258 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3259 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3260
3261 if (WaitStatesNeeded == MaxWaitStates)
3262 return WaitStatesNeeded; // Early exit.
3263
3264 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3265 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3266 return false;
3267 Register DstReg = MI.getOperand(0).getReg();
3268 return TRI.regsOverlap(Reg, DstReg);
3269 };
3270
3271 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3272 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3273 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3274 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3275 if (OpNo == SrcCIdx)
3276 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3277 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3278 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3279
3280 WaitStatesNeededForUse = NeedWaitStates -
3281 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
3282 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3283
3284 if (WaitStatesNeeded == MaxWaitStates)
3285 return WaitStatesNeeded; // Early exit.
3286 }
3287
3288 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3289 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3290 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3291 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3292 const int MaxWaitStates = 13;
3293 Register DstReg = MI->getOperand(0).getReg();
3294 unsigned HazardDefLatency = 0;
3295
3296 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3297 this](const MachineInstr &MI) {
3298 if (!SIInstrInfo::isMFMA(MI))
3299 return false;
3300 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
3301 HazardDefLatency =
3302 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
3303 return TRI.regsOverlap(Reg, DstReg);
3304 };
3305
3306 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
3307 int NeedWaitStates;
3308 switch (HazardDefLatency) {
3309 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3310 break;
3311 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3312 break;
3313 case 16: [[fallthrough]];
3314 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3315 break;
3316 }
3317
3318 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3319 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3320 }
3321
3322 // Pad neighboring MFMA with noops for better inter-wave performance.
3323 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
3324
3325 return WaitStatesNeeded;
3326}
3327
3328/// One MFMA can be written with up to four opcodes that differ only in how vdst
3329/// and src2 are encoded: both are either AGPRs or VGPRs, and the mac form ties
3330/// vdst to src2 instead of taking them as separate operands. \returns the AGPR,
3331/// non-mac opcode, so that every form of the same MFMA maps to one value.
3332static unsigned getMFMANonMacAGPRFormOp(unsigned Opc) {
3333 if (int NonMacOp = AMDGPU::getMFMAEarlyClobberOp(Opc); NonMacOp != -1)
3334 Opc = NonMacOp;
3335 if (int AGPROp = AMDGPU::getAGPRFormOp(Opc); AGPROp != -1)
3336 Opc = AGPROp;
3337 return Opc;
3338}
3339
3340/// \returns true if \p Opc0 and \p Opc1 are the same MFMA, ignoring the mac
3341/// form and whether vdst/src2 are AGPRs or VGPRs.
3342static bool isSameMFMA(unsigned Opc0, unsigned Opc1) {
3344}
3345
3346static int
3348 bool IsGFX950) {
3349 // xdl def cycles | gfx940 | gfx950
3350 // 2 pass | 3 4
3351 // 4 pass | 5 6
3352 // 8 pass | 9 10
3353 // 16 pass | 17 18
3354 return NumPasses + 1 + IsGFX950;
3355}
3356
3357static int
3359 bool IsGFX950) {
3360 // xdl def cycles | gfx940 | gfx950
3361 // 2 pass | 3 3
3362 // 4 pass | 5 6
3363 // 8 pass | 9 10
3364 // 16 pass | 17 18
3365 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3366}
3367
3368static int
3370 // 2 pass -> 2
3371 // 4 pass -> 4
3372 // 8 pass -> 8
3373 // 16 pass -> 16
3374 return NumPasses;
3375}
3376
3377static int
3379 // 2 pass -> 4
3380 // 4 pass -> 6
3381 // 8 pass -> 10
3382 // 16 pass -> 18
3383 return NumPasses + 2;
3384}
3385
3387 bool IsGFX950) {
3388 // xdl def cycles | gfx942 | gfx950
3389 // 2 pass | 5 5
3390 // 4 pass | 7 8
3391 // 8 pass | 11 12
3392 // 16 pass | 19 20
3393 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3394}
3395
3396int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
3397 const MachineInstr *Reader, const MachineInstr *Writer) const {
3398 constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3399 constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3400 constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3401 constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3402 constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3403 constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3404 constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3405 constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3406 constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3407
3408 // An XDL read of a non-XDL result needs no wait states. DGEMM is never XDL,
3409 // so this also covers the f64 writers handled below.
3410 if (TII.isXDL(*Reader) && !TII.isXDL(*Writer))
3411 return 0;
3412
3413 switch (Writer->getOpcode()) {
3414 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3415 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3416 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3417 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3418 return ST.hasGFX950Insts()
3419 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3420 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3421 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3422 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3423 return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3424 default:
3425 break;
3426 }
3427
3428 int NumPasses = TSchedModel.computeInstrLatency(Writer);
3429 if (ST.hasGFX940Insts()) {
3430 if (!TII.isXDL(*Writer))
3432 NumPasses);
3433 return TII.isXDL(*Reader)
3435 NumPasses, ST.hasGFX950Insts())
3437 NumPasses, ST.hasGFX950Insts());
3438 }
3439
3440 bool IsDGEMM = SIInstrInfo::isDGEMM(Reader->getOpcode());
3441 switch (NumPasses) {
3442 case 2:
3443 return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3444 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3445 case 8:
3446 return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3447 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3448 case 16:
3449 return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3450 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3451 default:
3452 llvm_unreachable("unexpected number of passes");
3453 }
3454}
3455
3456int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
3457 const MachineInstr &Producer,
3458 Register Reg,
3459 bool IsSrcC) const {
3460 unsigned Opc = Consumer.getOpcode();
3461 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3462 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3463 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3464 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3465 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3466 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3467 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3468 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3469
3470 bool FullReg = Producer.getOperand(0).getReg() == Reg;
3471 unsigned Opc1 = Producer.getOpcode();
3472 int NeedWaitStates = 0;
3473 if (IsSrcC) {
3474 if (!SIInstrInfo::isDGEMM(Opc) &&
3475 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
3476 NeedWaitStates = 0;
3477 } else if (FullReg) {
3478 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3479 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3480 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3481 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3482 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3483 else if (ST.hasGFX940Insts() &&
3484 TSchedModel.computeInstrLatency(&Producer) == 2)
3485 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3486
3487 // The accumulator forwarding path that allows zero wait states is only
3488 // available while the chain stays on a single MFMA. Two different MFMAs
3489 // sharing an accumulator need the wait states of a partial overlap.
3490 if (ST.hasGFX940Insts() && !isSameMFMA(Opc, Opc1)) {
3491 NeedWaitStates =
3492 std::max(NeedWaitStates,
3493 getMFMAOverlappedSrcCWaitStates(&Consumer, &Producer));
3494 }
3495 } else {
3496 NeedWaitStates = getMFMAOverlappedSrcCWaitStates(&Consumer, &Producer);
3497 }
3498 } else {
3499 switch (Opc1) {
3500 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3501 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3502 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3503 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3504 NeedWaitStates =
3505 ST.hasGFX950Insts()
3506 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3507 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3508 break;
3509 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3510 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3511 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3512 break;
3513 default:
3514 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3515
3516 if (ST.hasGFX940Insts()) {
3517 NeedWaitStates =
3518 TII.isXDL(Producer)
3520 NumPasses, ST.hasGFX950Insts())
3522 NumPasses);
3523 break;
3524 }
3525
3526 switch (NumPasses) {
3527 case 2:
3528 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3529 break;
3530 case 4:
3531 llvm_unreachable("unexpected number of passes for mfma");
3532 case 8:
3533 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3534 break;
3535 case 16:
3536 default:
3537 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3538 }
3539 }
3540 }
3541 return NeedWaitStates;
3542}
3543
3544int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3545 int WaitStatesNeeded = 0;
3546 unsigned Opc = MI->getOpcode();
3547
3548 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3549 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3551 };
3552
3553 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3554 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3556 };
3557
3558 if (!SIInstrInfo::isMFMA(*MI))
3559 return WaitStatesNeeded;
3560
3561 const int VALUWritesExecWaitStates = 4;
3562 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3563 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
3564 VALUWritesExecWaitStates);
3565 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3566
3567 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
3568
3569 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3570 for (const MachineOperand &Use : MI->explicit_uses()) {
3571 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3572 const int MaxWaitStates =
3574 16, ST.hasGFX950Insts());
3575
3576 if (!Use.isReg())
3577 continue;
3578 Register Reg = Use.getReg();
3579
3580 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3581 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
3582 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3583
3584 int OpNo = Use.getOperandNo();
3585 WaitStatesNeeded = std::max(
3586 WaitStatesNeeded,
3587 getMaxWindowDeficit(
3588 MaxWaitStates, [&](const MachineInstr &P) -> std::optional<int> {
3589 if (!SIInstrInfo::isMFMA(P) ||
3590 !TRI.regsOverlap(P.getOperand(0).getReg(), Reg))
3591 return std::nullopt;
3592 return getMFMAReadWaitStates(*MI, P, Reg, OpNo == SrcCIdx);
3593 }));
3594
3595 if (WaitStatesNeeded == MaxWaitStates)
3596 break;
3597 }
3598
3599 // Pad neighboring MFMA with noops for better inter-wave performance.
3600 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
3601
3602 return WaitStatesNeeded;
3603}
3604
3605int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3606 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3607 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3608 return 0;
3609
3610 int WaitStatesNeeded = 0;
3611
3612 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3613 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3614 };
3615
3616 for (const MachineOperand &Op : MI->explicit_uses()) {
3617 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
3618 continue;
3619
3620 Register Reg = Op.getReg();
3621
3622 const int AccVgprReadLdStWaitStates = 2;
3623 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3624 const int MaxWaitStates = 2;
3625
3626 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3627 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
3628 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3629
3630 if (WaitStatesNeeded == MaxWaitStates)
3631 return WaitStatesNeeded; // Early exit.
3632
3633 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3634 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3635 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3636 return false;
3637 auto IsVALUFn = [](const MachineInstr &MI) {
3638 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3640 };
3641 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
3642 std::numeric_limits<int>::max();
3643 };
3644
3645 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3646 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
3647 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3648 }
3649
3650 return WaitStatesNeeded;
3651}
3652
3653int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3654 assert(!ST.hasVcmpxPermlaneHazard() &&
3655 "this is a different vcmpx+permlane hazard");
3656 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3657 const SIInstrInfo *TII = ST.getInstrInfo();
3658
3659 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3660 return isVCmpXWritesExec(*TII, *TRI, MI);
3661 };
3662
3663 auto IsVALUFn = [](const MachineInstr &MI) {
3664 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
3665 };
3666
3667 const int VCmpXWritesExecWaitStates = 4;
3668 const int VALUWritesVDstWaitStates = 2;
3669 int WaitStatesNeeded = 0;
3670
3671 for (const MachineOperand &Op : MI->explicit_uses()) {
3672 if (!Op.isReg() || !TRI->isVGPR(MF.getRegInfo(), Op.getReg()))
3673 continue;
3674 Register Reg = Op.getReg();
3675
3676 int WaitStatesSinceDef =
3677 VALUWritesVDstWaitStates -
3678 getWaitStatesSinceDef(Reg, IsVALUFn,
3679 /*MaxWaitStates=*/VALUWritesVDstWaitStates);
3680 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesSinceDef);
3681 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3682 break;
3683 }
3684
3685 int VCmpXHazardWaits =
3686 VCmpXWritesExecWaitStates -
3687 getWaitStatesSince(IsVCmpXWritesExecFn, VCmpXWritesExecWaitStates);
3688
3689 WaitStatesNeeded = std::max(WaitStatesNeeded, VCmpXHazardWaits);
3690 return WaitStatesNeeded;
3691}
3692
3694 // 2 pass -> 4
3695 // 4 pass -> 6
3696 // 8 pass -> 10
3697 // 16 pass -> 18
3698 return NumPasses + 2;
3699}
3700
3702 bool IsGFX950) {
3703 // xdl def cycles | gfx942 | gfx950
3704 // 2 pass | 5 5
3705 // 4 pass | 7 8
3706 // 8 pass | 11 12
3707 // 16 pass | 19 20
3708 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3709}
3710
3712 bool IsGFX950) {
3713 // xdl def cycles | gfx942 | gfx950
3714 // 2 pass | 5 5
3715 // 4 pass | 7 8
3716 // 8 pass | 11 12
3717 // 16 pass | 19 20
3718 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3719}
3720
3722 // 2 pass -> 4
3723 // 4 pass -> 6
3724 // 8 pass -> 10
3725 // 16 pass -> 18
3726 return NumPasses + 2;
3727}
3728
3729int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3730 if (!ST.hasGFX90AInsts())
3731 return 0;
3732
3733 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3734 return SIInstrInfo::isDGEMM(MI.getOpcode());
3735 };
3736
3737 // This is checked in checkMAIHazards90A()
3738 if (SIInstrInfo::isMFMA(*MI))
3739 return 0;
3740
3741 const MachineRegisterInfo &MRI = MF.getRegInfo();
3742
3743 int WaitStatesNeeded = 0;
3744
3745 bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI);
3746 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
3747 bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false);
3748
3749 unsigned Reg;
3750 auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) {
3751 if (!SIInstrInfo::isMFMA(MI) ||
3752 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
3753 return false;
3754 return true;
3755 };
3756
3757 const MachineInstr *DOT = nullptr;
3758 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3759 if (!SIInstrInfo::isDOT(MI) ||
3760 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
3761 return false;
3762 DOT = &MI;
3763 return true;
3764 };
3765
3766 bool DGEMMAfterVALUWrite = false;
3767 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3768 // Found DGEMM on reverse traversal to def.
3769 if (SIInstrInfo::isDGEMM(MI.getOpcode()))
3770 DGEMMAfterVALUWrite = true;
3771
3772 // Only hazard if register is defined by a VALU and a DGEMM is found after
3773 // after the def.
3774 if (!TII.isVALU(MI, /*AllowLDSDMA=*/false) || !DGEMMAfterVALUWrite)
3775 return false;
3776
3777 return true;
3778 };
3779
3780 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
3781 AMDGPU::OpName::src2);
3782
3783 if (IsMemOrExport || IsVALU) {
3784 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3785 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3786 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3787 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3788 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3789 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3790 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3791 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3792 const int DotWriteSameDotReadSrcAB = 3;
3793 const int DotWriteDifferentVALURead = 3;
3794 const int DMFMABetweenVALUWriteVMEMRead = 2;
3795 const int MaxWaitStates =
3797 ST.hasGFX950Insts());
3798
3799 for (const MachineOperand &Use : MI->explicit_uses()) {
3800 if (!Use.isReg())
3801 continue;
3802 Reg = Use.getReg();
3803
3804 DOT = nullptr;
3805 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
3806 MaxWaitStates);
3807 if (DOT) {
3808 int NeedWaitStates = 0;
3809 if (DOT->getOpcode() == MI->getOpcode()) {
3810 if (&Use - &MI->getOperand(0) != SrcCIdx)
3811 NeedWaitStates = DotWriteSameDotReadSrcAB;
3812 } else {
3813 NeedWaitStates = DotWriteDifferentVALURead;
3814 }
3815
3816 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3817 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3818 }
3819
3820 // Workaround for HW data hazard bug observed only in GFX90A. When there
3821 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3822 // causes the SQ to incorrectly not insert two wait states between the two
3823 // instructions needed to avoid data hazard.
3824 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3825 DGEMMAfterVALUWrite = false;
3826 if (TRI.isVectorRegister(MRI, Reg)) {
3827 int WaitStatesNeededForUse =
3828 DMFMABetweenVALUWriteVMEMRead -
3829 getWaitStatesSinceDef(Reg, IsDGEMMHazard,
3830 DMFMABetweenVALUWriteVMEMRead);
3831
3832 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3833 }
3834 }
3835
3836 auto MFMAWindow = [&](const MachineInstr &Producer) {
3837 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3838 int NeedWaitStates = MaxWaitStates;
3839
3840 if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
3841 switch (NumPasses) {
3842 case 4:
3843 NeedWaitStates = IsMemOrExport
3844 ? DMFMA4x4WriteVgprMemExpReadWaitStates
3845 : DMFMA4x4WriteVgprVALUReadWaitStates;
3846 break;
3847 case 8:
3848 case 16:
3849 NeedWaitStates =
3850 IsMemOrExport
3851 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3852 : (ST.hasGFX950Insts()
3853 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3854 : DMFMA16x16WriteVgprVALUReadWaitStates);
3855 break;
3856 default:
3857 llvm_unreachable("unexpected dgemm");
3858 }
3859 } else if (ST.hasGFX940Insts()) {
3860 NeedWaitStates =
3861 TII.isXDL(Producer)
3863 NumPasses, ST.hasGFX950Insts())
3865 NumPasses);
3866 } else {
3867 switch (NumPasses) {
3868 case 2:
3869 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3870 break;
3871 case 8:
3872 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3873 break;
3874 case 16:
3875 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3876 break;
3877 default:
3878 llvm_unreachable("unexpected number of passes for mfma");
3879 }
3880 }
3881
3882 assert(NeedWaitStates <= MaxWaitStates &&
3883 "hazard requirement exceeds the scan window");
3884 return NeedWaitStates;
3885 };
3886
3887 WaitStatesNeeded = std::max(
3888 WaitStatesNeeded,
3889 getMaxWindowDeficit(MaxWaitStates,
3890 [&](const MachineInstr &P) -> std::optional<int> {
3891 if (!IsMFMAWriteFn(P))
3892 return std::nullopt;
3893 return MFMAWindow(P);
3894 }));
3895
3896 if (WaitStatesNeeded >= MaxWaitStates)
3897 break;
3898 }
3899 }
3900
3901 unsigned Opc = MI->getOpcode();
3902 const int DMFMAToFMA64WaitStates = 2;
3903 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3904 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3905 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3906 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3907 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3908 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
3909 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
3910 }
3911
3912 if (!IsVALU && !IsMemOrExport)
3913 return WaitStatesNeeded;
3914
3915 for (const MachineOperand &Def : MI->defs()) {
3916 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3917 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3918 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3919 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3920 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3921 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3922 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3923 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3924 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3925 const int DotWriteDifferentVALUWrite = 3;
3926 const int MaxWaitStates =
3927 GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(16, ST.hasGFX950Insts());
3928 const int MaxWarWaitStates = 15;
3929
3930 Reg = Def.getReg();
3931
3932 DOT = nullptr;
3933 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
3934 MaxWaitStates);
3935 if (DOT && DOT->getOpcode() != MI->getOpcode())
3936 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
3937 WaitStatesSinceDef);
3938
3939 auto MFMAWawWindow = [&](const MachineInstr &Producer) {
3940 int NeedWaitStates = MaxWaitStates;
3941 int NumPasses = TSchedModel.computeInstrLatency(&Producer);
3942
3943 if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
3944 switch (NumPasses) {
3945 case 4:
3946 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3947 break;
3948 case 8:
3949 case 16:
3950 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3951 break;
3952 default:
3953 llvm_unreachable("unexpected number of cycles for dgemm");
3954 }
3955 } else if (ST.hasGFX940Insts()) {
3956 NeedWaitStates =
3957 TII.isXDL(Producer)
3959 NumPasses, ST.hasGFX950Insts())
3961 } else {
3962 switch (NumPasses) {
3963 case 2:
3964 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3965 break;
3966 case 8:
3967 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3968 break;
3969 case 16:
3970 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3971 break;
3972 default:
3973 llvm_unreachable("Unexpected number of passes for mfma");
3974 }
3975 }
3976
3977 assert(NeedWaitStates <= MaxWaitStates &&
3978 "hazard requirement exceeds the scan window");
3979 return NeedWaitStates;
3980 };
3981
3982 WaitStatesNeeded = std::max(
3983 WaitStatesNeeded,
3984 getMaxWindowDeficit(MaxWaitStates,
3985 [&](const MachineInstr &P) -> std::optional<int> {
3986 if (!IsMFMAWriteFn(P))
3987 return std::nullopt;
3988 return MFMAWawWindow(P);
3989 }));
3990
3991 if (WaitStatesNeeded >= MaxWaitStates)
3992 break;
3993
3994 // Wait states this write needs after an MFMA that read Reg as srcC.
3995 auto SMFMAReadAsCWindow =
3996 [&](const MachineInstr &Reader) -> std::optional<int> {
3997 if (!SIInstrInfo::isMFMA(Reader) ||
3998 SIInstrInfo::isDGEMM(Reader.getOpcode()) ||
3999 !Reader.readsRegister(Reg, &TRI))
4000 return std::nullopt;
4001
4002 if (ST.hasGFX940Insts() && !TII.isXDL(Reader))
4003 return std::nullopt;
4004
4005 const MachineOperand *SrcC =
4006 TII.getNamedOperand(Reader, AMDGPU::OpName::src2);
4007 assert(SrcC && "an MFMA must have a src2 operand");
4008 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
4009 return std::nullopt;
4010
4011 switch (TSchedModel.computeInstrLatency(&Reader)) {
4012 case 2:
4013 return SMFMA4x4ReadVgprVALUWarWaitStates;
4014 case 4:
4015 assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later");
4016 return GFX940_XDL4PassReadVgprVALUWarWaitStates;
4017 case 8:
4018 return SMFMA16x16ReadVgprVALUWarWaitStates;
4019 case 16:
4020 [[fallthrough]];
4021 default:
4022 return SMFMA32x32ReadVgprVALUWarWaitStates;
4023 }
4024 };
4025
4026 WaitStatesNeeded =
4027 std::max(WaitStatesNeeded,
4028 getMaxWindowDeficit(MaxWarWaitStates, SMFMAReadAsCWindow));
4029 }
4030
4031 return WaitStatesNeeded;
4032}
4033
4035 if (!SU->isInstr())
4036 return false;
4037
4038 const MachineInstr *MAI = nullptr;
4039
4040 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
4041 MAI = nullptr;
4043 MAI = &MI;
4044 return MAI != nullptr;
4045 };
4046
4047 MachineInstr *MI = SU->getInstr();
4048 if (IsMFMAFn(*MI)) {
4049 int W = getWaitStatesSince(IsMFMAFn, 16);
4050 if (MAI)
4051 return W < (int)TSchedModel.computeInstrLatency(MAI);
4052 }
4053
4054 return false;
4055}
4056
4057// Adjust global offsets for instructions bundled with S_GETPC_B64 after
4058// insertion of a new instruction.
4059static void updateGetPCBundle(MachineInstr *NewMI) {
4060 if (!NewMI->isBundled())
4061 return;
4062
4063 // Find start of bundle.
4064 auto I = NewMI->getIterator();
4065 while (I->isBundledWithPred())
4066 I--;
4067 if (I->isBundle())
4068 I++;
4069
4070 // Bail if this is not an S_GETPC bundle.
4071 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
4072 return;
4073
4074 // Update offsets of any references in the bundle.
4075 const unsigned NewBytes = 4;
4076 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4077 "Unexpected instruction insertion in bundle");
4078 auto NextMI = std::next(NewMI->getIterator());
4079 auto End = NewMI->getParent()->end();
4080 while (NextMI != End && NextMI->isBundledWithPred()) {
4081 for (auto &Operand : NextMI->operands()) {
4082 if (Operand.isGlobal())
4083 Operand.setOffset(Operand.getOffset() + NewBytes);
4084 }
4085 NextMI++;
4086 }
4087}
4088
4089bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
4090 if (!ST.hasVALUMaskWriteHazard())
4091 return false;
4092 assert(!ST.hasExtendedWaitCounts());
4093
4094 if (!ST.isWave64())
4095 return false;
4096
4097 const bool IsSALU = SIInstrInfo::isSALU(*MI);
4098 const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false);
4099 if (!IsSALU && !IsVALU)
4100 return false;
4101
4102 // The hazard sequence is three instructions:
4103 // 1. VALU reads SGPR as mask
4104 // 2. VALU/SALU writes SGPR
4105 // 3. VALU/SALU reads SGPR
4106 // The hazard can expire if the distance between 2 and 3 is sufficient,
4107 // or (2) is VALU and (3) is SALU.
4108 // In practice this happens <10% of the time, hence always assume the hazard
4109 // exists if (1) and (2) are present to avoid searching all SGPR reads.
4110
4111 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4112 const MachineRegisterInfo &MRI = MF.getRegInfo();
4113
4114 auto IgnoreableSGPR = [](const Register Reg) {
4115 switch (Reg) {
4116 case AMDGPU::EXEC:
4117 case AMDGPU::EXEC_LO:
4118 case AMDGPU::EXEC_HI:
4119 case AMDGPU::M0:
4120 case AMDGPU::SGPR_NULL:
4121 case AMDGPU::SGPR_NULL64:
4122 case AMDGPU::SCC:
4123 return true;
4124 default:
4125 return false;
4126 }
4127 };
4128 auto IsVCC = [](const Register Reg) {
4129 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
4130 };
4131
4132 struct StateType {
4133 SmallSet<Register, 2> HazardSGPRs;
4134
4135 static unsigned getHashValue(const StateType &State) {
4136 return static_cast<unsigned>(hash_combine_range(State.HazardSGPRs));
4137 }
4138 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4139 return LHS.HazardSGPRs == RHS.HazardSGPRs;
4140 }
4141 };
4142
4143 StateType InitialState;
4144
4145 // Look for SGPR write.
4146 MachineOperand *HazardDef = nullptr;
4147 for (MachineOperand &Op : MI->all_defs()) {
4148 Register Reg = Op.getReg();
4149 if (IgnoreableSGPR(Reg))
4150 continue;
4151 if (!IsVCC(Reg)) {
4152 if (Op.isImplicit())
4153 continue;
4154 if (!TRI->isSGPRReg(MRI, Reg))
4155 continue;
4156 }
4157
4158 HazardDef = &Op;
4159 break;
4160 }
4161
4162 if (!HazardDef)
4163 return false;
4164
4165 // Setup to track writes to individual SGPRs
4166 const Register HazardReg = HazardDef->getReg();
4167 if (AMDGPU::SReg_32RegClass.contains(HazardReg)) {
4168 InitialState.HazardSGPRs.insert(HazardReg);
4169 } else {
4170 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
4171 InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub0));
4172 InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub1));
4173 }
4174
4175 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4176 if (State.HazardSGPRs.empty())
4177 return HazardExpired;
4178
4179 switch (I.getOpcode()) {
4180 case AMDGPU::V_ADDC_U32_e32:
4181 case AMDGPU::V_ADDC_U32_dpp:
4182 case AMDGPU::V_CNDMASK_B16_t16_e32:
4183 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4184 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4185 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4186 case AMDGPU::V_CNDMASK_B32_e32:
4187 case AMDGPU::V_CNDMASK_B32_dpp:
4188 case AMDGPU::V_DIV_FMAS_F32_e64:
4189 case AMDGPU::V_DIV_FMAS_F64_e64:
4190 case AMDGPU::V_SUBB_U32_e32:
4191 case AMDGPU::V_SUBB_U32_dpp:
4192 case AMDGPU::V_SUBBREV_U32_e32:
4193 case AMDGPU::V_SUBBREV_U32_dpp: {
4194 // These implicitly read VCC as mask source.
4195 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4196 }
4197 case AMDGPU::V_ADDC_U32_e64:
4198 case AMDGPU::V_ADDC_U32_e64_dpp:
4199 case AMDGPU::V_CNDMASK_B16_t16_e64:
4200 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4201 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4202 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4203 case AMDGPU::V_CNDMASK_B32_e64:
4204 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4205 case AMDGPU::V_SUBB_U32_e64:
4206 case AMDGPU::V_SUBB_U32_e64_dpp:
4207 case AMDGPU::V_SUBBREV_U32_e64:
4208 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4209 // Only check mask register overlaps.
4210 const MachineOperand *SSRCOp = TII.getNamedOperand(I, AMDGPU::OpName::src2);
4211 assert(SSRCOp);
4212 bool Result = TRI->regsOverlap(SSRCOp->getReg(), HazardReg);
4213 return Result ? HazardFound : NoHazardFound;
4214 }
4215 default:
4216 return NoHazardFound;
4217 }
4218 };
4219
4220 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4221 // Update tracking of SGPR writes.
4222 for (auto &Op : I.all_defs()) {
4223 Register Reg = Op.getReg();
4224 if (IgnoreableSGPR(Reg))
4225 continue;
4226 if (!IsVCC(Reg)) {
4227 if (Op.isImplicit())
4228 continue;
4229 if (!TRI->isSGPRReg(MRI, Reg))
4230 continue;
4231 }
4232
4233 // Stop tracking any SGPRs with writes on the basis that they will
4234 // already have an appropriate wait inserted afterwards.
4236 for (Register SGPR : State.HazardSGPRs) {
4237 if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
4238 Found.push_back(SGPR);
4239 }
4240 for (Register SGPR : Found)
4241 State.HazardSGPRs.erase(SGPR);
4242 }
4243 };
4244
4245 // Check for hazard
4246 if (!hasHazard<StateType>(InitialState, IsHazardFn, UpdateStateFn,
4247 MI->getParent(),
4248 std::next(MI->getReverseIterator())))
4249 return false;
4250
4251 // Compute counter mask
4252 unsigned DepCtr =
4253 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(0, ST)
4254 : AMDGPU::DepCtr::encodeFieldVaSdst(0, ST))
4255 : AMDGPU::DepCtr::encodeFieldSaSdst(0, ST);
4256
4257 // Add s_waitcnt_depctr after SGPR write.
4258 auto NextMI = std::next(MI->getIterator());
4259 auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
4260 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
4261 .addImm(DepCtr);
4262
4263 // SALU write may be s_getpc in a bundle.
4264 updateGetPCBundle(NewMI);
4265
4266 return true;
4267}
4268
4269static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4270 const SIInstrInfo &TII) {
4271 MachineBasicBlock &EntryMBB = MF->front();
4272 if (EntryMBB.begin() != EntryMBB.end()) {
4273 auto &EntryMI = *EntryMBB.begin();
4274 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4275 EntryMI.getOperand(0).getImm() >= Priority)
4276 return false;
4277 }
4278
4279 BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), TII.get(AMDGPU::S_SETPRIO))
4280 .addImm(Priority);
4281 return true;
4282}
4283
4284bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4285 if (!ST.hasRequiredExportPriority())
4286 return false;
4287
4288 // Assume the following shader types will never have exports,
4289 // and avoid adding or adjusting S_SETPRIO.
4290 MachineBasicBlock *MBB = MI->getParent();
4291 MachineFunction *MF = MBB->getParent();
4292 auto CC = MF->getFunction().getCallingConv();
4293 switch (CC) {
4298 return false;
4299 default:
4300 break;
4301 }
4302
4303 const int MaxPriority = 3;
4304 const int NormalPriority = 2;
4305 const int PostExportPriority = 0;
4306
4307 auto It = MI->getIterator();
4308 switch (MI->getOpcode()) {
4309 case AMDGPU::S_ENDPGM:
4310 case AMDGPU::S_ENDPGM_SAVED:
4311 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4312 case AMDGPU::SI_RETURN_TO_EPILOG:
4313 // Ensure shader with calls raises priority at entry.
4314 // This ensures correct priority if exports exist in callee.
4315 if (MF->getFrameInfo().hasCalls())
4316 return ensureEntrySetPrio(MF, NormalPriority, TII);
4317 return false;
4318 case AMDGPU::S_SETPRIO: {
4319 // Raise minimum priority unless in workaround.
4320 auto &PrioOp = MI->getOperand(0);
4321 int Prio = static_cast<int>(PrioOp.getImm());
4322 bool InWA = (Prio == PostExportPriority) &&
4323 (It != MBB->begin() && TII.isEXP(*std::prev(It)));
4324 if (InWA || Prio >= NormalPriority)
4325 return false;
4326 PrioOp.setImm(std::min(Prio + NormalPriority, MaxPriority));
4327 return true;
4328 }
4329 default:
4330 if (!TII.isEXP(*MI))
4331 return false;
4332 break;
4333 }
4334
4335 // Check entry priority at each export (as there will only be a few).
4336 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4337 bool Changed = false;
4339 Changed = ensureEntrySetPrio(MF, NormalPriority, TII);
4340
4341 auto NextMI = std::next(It);
4342 bool EndOfShader = false;
4343 if (NextMI != MBB->end()) {
4344 // Only need WA at end of sequence of exports.
4345 if (TII.isEXP(*NextMI))
4346 return Changed;
4347 // Assume appropriate S_SETPRIO after export means WA already applied.
4348 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4349 NextMI->getOperand(0).getImm() == PostExportPriority)
4350 return Changed;
4351 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4352 }
4353
4354 const DebugLoc &DL = MI->getDebugLoc();
4355
4356 // Lower priority.
4357 BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO))
4358 .addImm(PostExportPriority);
4359
4360 if (!EndOfShader) {
4361 // Wait for exports to complete.
4362 BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_WAITCNT_EXPCNT))
4363 .addReg(AMDGPU::SGPR_NULL)
4364 .addImm(0);
4365 }
4366
4367 BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0);
4368 BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0);
4369
4370 if (!EndOfShader) {
4371 // Return to normal (higher) priority.
4372 BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO))
4373 .addImm(NormalPriority);
4374 }
4375
4376 return true;
4377}
4378
4379// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
4380// next instruction that actually issues. Unlike skipDebugInstructionsForward /
4381// next_nodbg, this skips the full isMetaInstruction() set.
4385 while (I != End && I->isMetaInstruction())
4386 ++I;
4387 return I;
4388}
4389
4390bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
4391 // Requirement #1 of 2:
4392 // The cross-wave entry-block mitigation is delegated to the mandatory
4393 // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
4394 assert(ST.hasRequiresInitialUnclausedVmem() &&
4395 "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
4396 "prologue to satisfy the cross-wave entry mitigation");
4397
4398 if (!SIInstrInfo::isVPermPk16(MI->getOpcode()))
4399 return false;
4400
4401 MachineBasicBlock *MBB = MI->getParent();
4402
4403 // Requirement #2 of 2:
4404 // V_PERM_PK16 must be immediately followed by a safe instruction.
4406 skipMetaInstructionsForward(std::next(MI->getIterator()), MBB->end());
4407 if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(*NextI))
4408 return false;
4409
4410 // EXEC is guaranteed non-zero here: V_PERM_PK16 reports unwanted effects
4411 // when EXEC is empty, so s_cbranch_execz over this region is retained.
4412 // A plain V_NOP is therefore a real VALU nop and clears the hazard.
4413 BuildMI(*MBB, NextI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4414 return true;
4415}
4416
4417bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4418 if (!isSGetReg(MI->getOpcode()))
4419 return false;
4420
4421 const SIInstrInfo *TII = ST.getInstrInfo();
4422 switch (getHWReg(TII, *MI)) {
4423 default:
4424 return false;
4429 break;
4430 }
4431
4432 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4433 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4434 .addImm(0);
4435 return true;
4436}
4437
4438bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4439 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4440 return false;
4441
4442 const SIInstrInfo *TII = ST.getInstrInfo();
4443 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4444 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4446 BuildMI(*MI->getParent(), std::next(MI->getIterator()), MI->getDebugLoc(),
4447 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4449
4450 return true;
4451}
4452
4453bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4454 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4455 // for hazard to trigger.
4457 return false;
4458
4459 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4460 const SIInstrInfo *TII = ST.getInstrInfo();
4461 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4462 const int FlatScrBaseWaitStates = 10;
4463
4464 bool ReadsFlatScrLo =
4465 MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4466 bool ReadsFlatScrHi =
4467 MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4468 if (isSGetReg(MI->getOpcode())) {
4469 switch (getHWReg(TII, *MI)) {
4470 default:
4471 break;
4473 ReadsFlatScrLo = true;
4474 break;
4476 ReadsFlatScrHi = true;
4477 break;
4478 }
4479 }
4480
4481 const MachineRegisterInfo &MRI = MF.getRegInfo();
4482
4483 auto IsRegDefHazard = [&](Register Reg) -> bool {
4484 DenseSet<const MachineBasicBlock *> Visited;
4485 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4486 return MI.modifiesRegister(Reg, TRI);
4487 };
4488
4489 // This literally abuses the idea of waitstates. Instead of waitstates it
4490 // returns 1 for SGPR written and 0 otherwise.
4491 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4492 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/false))
4493 return 0;
4494 for (const MachineOperand &MO : MI.all_defs()) {
4495 if (TRI->isSGPRReg(MRI, MO.getReg()))
4496 return 1;
4497 }
4498 return 0;
4499 };
4500
4501 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4502 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4503 unsigned Wait = static_cast<unsigned>(MI.getOperand(0).getImm());
4506 return true;
4507 }
4508 return SgprWrites >= FlatScrBaseWaitStates;
4509 };
4510
4511 return ::getWaitStatesSince(
4512 IsHazardFn, MI->getParent(), std::next(MI->getReverseIterator()),
4513 0, IsExpiredFn, Visited, IsSGPRDef) < FlatScrBaseWaitStates;
4514 };
4515
4516 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(AMDGPU::SGPR102) ||
4517 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4518 (!ReadsFlatScrHi || MRI.isConstantPhysReg(AMDGPU::SGPR103) ||
4519 !IsRegDefHazard(AMDGPU::SGPR103)))
4520 return false;
4521
4522 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4523 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
4526 return true;
4527}
4528
4529bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4530 if (!isSSetReg(MI->getOpcode()) ||
4531 MI->getOperand(1).getImm() != AMDGPU::Hwreg::ID_MODE)
4532 return false;
4533
4534 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4535 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
4536 return true;
4537}
4538
4539bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4540 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4542 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4543 };
4544
4545 if (!IsTDM(*MI))
4546 return false;
4547
4548 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4549 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4550 return false;
4551 return MI.getOperand(0).getImm() <= 10;
4552 };
4553
4554 if (::getWaitStatesSince(IsTDM, MI, IsExpiredFn) ==
4555 std::numeric_limits<int>::max())
4556 return false;
4557
4558 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4559 TII.get(AMDGPU::S_WAIT_TENSORCNT))
4560 .addImm(10);
4561 return true;
4562}
4563
4564bool GCNHazardRecognizer::fixTransF16ToPartialVGPRWriteHazard(
4565 MachineInstr *MI) {
4566 if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false))
4567 return false;
4568
4569 const int NumVALUNeeded = 4;
4570 const MachineRegisterInfo &MRI = MF.getRegInfo();
4571
4572 struct StateType {
4573 int VALUs = 0;
4574
4575 static unsigned getHashValue(const StateType &State) { return State.VALUs; }
4576 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4577 return LHS.VALUs == RHS.VALUs;
4578 }
4579 };
4580
4581 StateType State;
4582
4583 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4584 // Too many VALU states have passed
4585 if (State.VALUs >= NumVALUNeeded)
4586 return HazardExpired;
4587
4588 // Instructions which cause va_vdst==0 expire hazard
4589 if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4591 static_cast<unsigned>(I.getOperand(0).getImm())) == 0)
4592 return HazardExpired;
4593
4594 // Track registers writes
4595 if (!SIInstrInfo::isTRANS(I))
4596 return NoHazardFound;
4597
4598 MCRegister Def = I.getOperand(0).getReg();
4599 if (!TRI.isVGPR(MRI, Def))
4600 return NoHazardFound;
4601
4602 unsigned Opc = I.getOpcode();
4603 const MCInstrDesc &Desc = TII.get(Opc);
4604 // Note: checking vdst does not work because it is a generic
4605 // OPERAND_REGISTER.
4606 switch (
4607 Desc.operands()[AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0)]
4608 .OperandType) {
4614 break;
4615 default:
4616 // DPP instructions have VGPR-only src0 with generic OPERAND_REGISTER
4617 // type. The isDPP() check will catch any TRANS opcodes including f32.
4618 // This is suboptimal but shall only affect fake16 compilation because
4619 // of the Def size check below.
4620 if (SIInstrInfo::isDPP(I))
4621 break;
4622 return NoHazardFound;
4623 }
4624
4625 if (TRI.getRegSizeInBits(Def, MRI) == 16) {
4626 // Real True16 case.
4627 const auto Sub = AMDGPU::isHi16Reg(Def, TRI)
4628 ? std::make_pair(AMDGPU::hi16, AMDGPU::lo16)
4629 : std::make_pair(AMDGPU::lo16, AMDGPU::hi16);
4630 MCRegister Def32 =
4631 TRI.getMatchingSuperReg(Def, Sub.first, &AMDGPU::VGPR_32RegClass);
4632 MCRegister OffendingDef = TRI.getSubReg(Def32, Sub.second);
4633 // Only interested in partial 16-bit write, not any wider register.
4634 return MI->modifiesRegister(OffendingDef, nullptr) ? HazardFound
4635 : NoHazardFound;
4636 }
4637
4638 // Bail for any non-16 bit DPP instructions catched in true16 mode.
4639 if (ST.useRealTrue16Insts())
4640 return NoHazardFound;
4641
4642 // Fake16 case is suboptimal as it checks the whole 32-bit def overlap w/o
4643 // analyzing OPSEL bits.
4644 return MI->modifiesRegister(Def, nullptr) ? HazardFound : NoHazardFound;
4645 };
4646
4647 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
4648 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
4649 ++State.VALUs;
4650 };
4651
4652 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
4653 std::next(MI->getReverseIterator())))
4654 return false;
4655
4656 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4657 TII.get(AMDGPU::S_WAITCNT_DEPCTR))
4659
4660 return true;
4661}
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t Imm
The AMDGPU TargetMachine interface definition for hw codegen targets.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(AttributeSet CallerAttrs, AttributeSet CalleeAttrs)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST)
static bool consumesDstSelForwardingOperand(const MachineInstr *VALU, const MachineOperand *Dst, const SIRegisterInfo *TRI)
Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .
static bool isSGetReg(unsigned Opcode)
static bool breaksSMEMSoftClause(MachineInstr *MI)
static bool isLdsDma(const MachineInstr &MI)
static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses, bool IsGFX950)
static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI, const SIInstrInfo *TII, const TargetSchedModel &SchedModel, const GCNSubtarget &ST)
static bool isRFE(unsigned Opcode)
static bool isRWLane(unsigned Opcode)
static bool isSMovRel(unsigned Opcode)
static unsigned getMFMANonMacAGPRFormOp(unsigned Opc)
One MFMA can be written with up to four opcodes that differ only in how vdst and src2 are encoded: bo...
#define DEBUG_TYPE_VERBOSE
static const MachineOperand * getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST)
Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value i...
static int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses, bool IsGFX950)
static void updateGetPCBundle(MachineInstr *NewMI)
static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses, bool IsGFX950)
static bool isStoreCountWaitZero(const MachineInstr &I)
static bool breaksVMEMSoftClause(MachineInstr *MI)
static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool isSSetReg(unsigned Opcode)
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
static bool isSameMFMA(unsigned Opc0, unsigned Opc1)
static bool isDivFMas(unsigned Opcode)
static unsigned getCountedWaitStates(const MachineInstr &MI, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates)
static bool hasHazard(StateT InitialState, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *InitialMBB, MachineBasicBlock::const_reverse_instr_iterator InitialI)
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses)
static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses, bool IsGFX950)
static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses)
static MachineBasicBlock::iterator skipMetaInstructionsForward(MachineBasicBlock::iterator I, MachineBasicBlock::iterator End)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses)
static bool isCoexecutableVALUInst(const MachineInstr &MI)
static bool ensureEntrySetPrio(MachineFunction *MF, int Priority, const SIInstrInfo &TII)
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred, const MachineBasicBlock *MBB)
static bool isPermlane(const MachineInstr &MI)
static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses)
static int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses, bool IsGFX950)
AMD GCN specific subclass of TargetSubtarget.
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static llvm::Error parse(GsymDataExtractor &Data, uint64_t BaseAddr, LineEntryCallback const &Callback)
Definition LineTable.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define H(x, y, z)
Definition MD5.cpp:56
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
Func MI getDebugLoc()))
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
unsigned get(InstCounterType T) const
BitVector & set()
Set all bits in the bitvector.
Definition BitVector.h:367
A debug info location.
Definition DebugLoc.h:126
std::pair< iterator, bool > insert_as(std::pair< KeyT, ValueT > &&KV, const LookupKeyT &Val)
Alternate version of insert() which allows a different, and possibly less expensive,...
Definition DenseMap.h:874
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
unsigned getHazardWaitStates(MachineInstr *MI) const
Returns the number of wait states until all hazards for MI are resolved.
unsigned PreEmitNoopsCommon(MachineInstr *) const
OperatingMode
Operating mode for the hazard recognizer.
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
function_ref< bool(const MachineInstr &)> IsHazardFn
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
function_ref< unsigned int(const MachineInstr &)> GetNumWaitStatesFn
bool ShouldPreferAnother(SUnit *SU) const override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
bool hasPhysRegs() const
Returns true if instruction operands are physical registers, so that hazards defined by register depe...
function_ref< bool(const MachineInstr &, int WaitStates)> IsExpiredFn
bool isSchedulerMode() const
Returns true if running as a scheduler (pre-RA or post-RA).
GCNHazardRecognizer(const MachineFunction &MF, OperatingMode Mode, MachineLoopInfo *MLI=nullptr)
Construct with explicit operating mode.
static AMDGPU::CoExecMaskT getCoExecMaskForMI(const MachineInstr &MI, const SIInstrInfo &TII)
Get the CoExecMask for a given instruction.
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
bool isHazardRecognizerMode() const
Returns true if running as the standalone hazard recognizer pass.
bool isPreRA() const
Returns true if running in pre-RA scheduling mode.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
Instructions::const_reverse_iterator const_reverse_instr_iterator
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Instructions::iterator instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool isBundled() const
Return true if this instruction part of a bundle.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
void setIsKill(bool Val=true)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool isConstantPhysReg(MCRegister PhysReg) const
Returns true if PhysReg is unallocatable and constant throughout the function.
LLVM_ABI bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
static bool isDS(const MachineInstr &MI)
static bool isVMEM(const MachineInstr &MI)
static bool isSMRD(const MachineInstr &MI)
static bool isMTBUF(const MachineInstr &MI)
static bool isDGEMM(unsigned Opcode)
static bool isEXP(const MachineInstr &MI)
static bool isSALU(const MachineInstr &MI)
static bool isSDWA(const MachineInstr &MI)
static bool isDOT(const MachineInstr &MI)
static bool usesTENSOR_CNT(const MachineInstr &MI)
static bool isSWMMAC(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isVPermPk16(unsigned Opcode)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
static bool isTRANS(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
static bool isWaitcnt(unsigned Opcode)
static bool isVOP3(const MCInstrDesc &Desc)
static bool isDPP(const MachineInstr &MI)
static bool isMFMA(const MachineInstr &MI)
static bool isMAI(const MCInstrDesc &Desc)
static bool isFPAtomic(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
static bool isVOP3P(const MachineInstr &MI)
static bool isWMMA(const MachineInstr &MI)
static bool isLDSDMA(const MachineInstr &MI)
Scheduling unit. This is a node in the scheduling DAG.
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
virtual void EmitNoops(unsigned Quantity)
EmitNoops - This callback is invoked when noops were added to the instruction stream.
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:346
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition StringRef.h:482
Provide an instruction scheduling machine model to CodeGen passes.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
Definition ilist_node.h:123
A range adaptor for a pair of iterators.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc)
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst)
unsigned decodeFieldSaSdst(unsigned Encoded)
unsigned decodeFieldVaSdst(unsigned Encoded)
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc)
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst)
unsigned decodeFieldVaVdst(unsigned Encoded)
unsigned decodeFieldVmVsrc(unsigned Encoded)
unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst)
const char * getCoExecMaskName(CoExecMaskT Mask)
Return a human-readable name for a mask holding a single instruction class, as produced by getCoExecM...
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
constexpr unsigned MaxCoExecStages
Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
FPType getFPDstSelType(unsigned Opc)
bool isGFX12Plus(const MCSubtargetInfo &STI)
StringRef getSchedStrategy(const AMDGPUOptions &CLOpts, const Function &F)
const char * getStageTypeName(CoExecStageType T)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
LLVM_READONLY int32_t getMFMAEarlyClobberOp(uint32_t Opcode)
CoExecMask CoExecMaskT
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded)
CoExecInfo getCoExecInfo(const MachineInstr &MI, const SIInstrInfo &TII)
Get co-execution info for a WMMA instruction, selecting the per-cycle slot pattern from the opcode (a...
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
LLVM_READONLY int32_t getAGPRFormOp(uint32_t Opcode)
InstructionFlavor classifyFlavor(const MachineInstr &MI, const SIInstrInfo &SII)
Classify MI into the execution flavor that drives both the scheduler's slot preferences and the hazar...
@ OPERAND_REG_IMM_NOINLINE_FP16
Definition SIDefines.h:438
@ OPERAND_REG_INLINE_C_BF16
Definition SIDefines.h:453
@ OPERAND_REG_IMM_BF16
Definition SIDefines.h:436
@ OPERAND_REG_IMM_FP16
Definition SIDefines.h:437
@ OPERAND_REG_INLINE_C_FP16
Definition SIDefines.h:454
CoExecMaskT getCoExecMask(InstructionFlavor F)
Map a flavor to the co-execution class it occupies in a window slot.
bool isGFX1250(const MCSubtargetInfo &STI)
@ Entry
Definition COFF.h:862
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
This namespace contains all of the command line option processing machinery.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
constexpr double e
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
@ Wait
Definition Threading.h:60
constexpr RegState getDeadRegState(bool B)
Op::Description Desc
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
@ Sub
Subtraction of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:307
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:287
#define N
Co-execution characteristics for a multi-cycle instruction.
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
An information struct used to provide DenseMap with the various necessary components for a given valu...