LLVM  15.0.0git
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
16 #include "SIMachineFunctionInfo.h"
20 
21 using namespace llvm;
22 
23 namespace {
24 
25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
26  MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
27 
28  bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
29  if (Arg.getAsInteger(0, Value))
30  return O.error("'" + Arg + "' value invalid for uint argument!");
31 
32  if (Value > 100)
33  return O.error("'" + Arg + "' value must be in the range [0, 100]!");
34 
35  return false;
36  }
37 };
38 
39 } // end anonymous namespace
40 
42  MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
43  cl::desc("Fill a percentage of the latency between "
44  "neighboring MFMA with s_nops."));
45 
46 //===----------------------------------------------------------------------===//
47 // Hazard Recognizer Implementation
48 //===----------------------------------------------------------------------===//
49 
51  const GCNSubtarget &ST);
52 
54  IsHazardRecognizerMode(false),
55  CurrCycleInstr(nullptr),
56  MF(MF),
57  ST(MF.getSubtarget<GCNSubtarget>()),
58  TII(*ST.getInstrInfo()),
59  TRI(TII.getRegisterInfo()),
60  ClauseUses(TRI.getNumRegUnits()),
61  ClauseDefs(TRI.getNumRegUnits()) {
62  MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
63  TSchedModel.init(&ST);
64  RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
65 }
66 
68  EmittedInstrs.clear();
69 }
70 
73 }
74 
76  CurrCycleInstr = MI;
77 }
78 
79 static bool isDivFMas(unsigned Opcode) {
80  return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
81 }
82 
83 static bool isSGetReg(unsigned Opcode) {
84  return Opcode == AMDGPU::S_GETREG_B32;
85 }
86 
87 static bool isSSetReg(unsigned Opcode) {
88  switch (Opcode) {
89  case AMDGPU::S_SETREG_B32:
90  case AMDGPU::S_SETREG_B32_mode:
91  case AMDGPU::S_SETREG_IMM32_B32:
92  case AMDGPU::S_SETREG_IMM32_B32_mode:
93  return true;
94  }
95  return false;
96 }
97 
98 static bool isRWLane(unsigned Opcode) {
99  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
100 }
101 
102 static bool isRFE(unsigned Opcode) {
103  return Opcode == AMDGPU::S_RFE_B64;
104 }
105 
106 static bool isSMovRel(unsigned Opcode) {
107  switch (Opcode) {
108  case AMDGPU::S_MOVRELS_B32:
109  case AMDGPU::S_MOVRELS_B64:
110  case AMDGPU::S_MOVRELD_B32:
111  case AMDGPU::S_MOVRELD_B64:
112  return true;
113  default:
114  return false;
115  }
116 }
117 
118 static bool isDGEMM(unsigned Opcode) {
119  return AMDGPU::getMAIIsDGEMM(Opcode);
120 }
121 
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
123  unsigned Opcode = MI.getOpcode();
124 
125  if (!SIInstrInfo::isMAI(MI) ||
126  isDGEMM(Opcode) ||
127  Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
128  Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
129  return false;
130 
131  if (!ST.hasGFX940Insts())
132  return true;
133 
134  return AMDGPU::getMAIIsGFX940XDL(Opcode);
135 }
136 
138  const MachineInstr &MI) {
139  if (TII.isAlwaysGDS(MI.getOpcode()))
140  return true;
141 
142  switch (MI.getOpcode()) {
143  case AMDGPU::S_SENDMSG:
144  case AMDGPU::S_SENDMSGHALT:
145  case AMDGPU::S_TTRACEDATA:
146  return true;
147  // These DS opcodes don't support GDS.
148  case AMDGPU::DS_NOP:
149  case AMDGPU::DS_PERMUTE_B32:
150  case AMDGPU::DS_BPERMUTE_B32:
151  return false;
152  default:
153  if (TII.isDS(MI.getOpcode())) {
154  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
155  AMDGPU::OpName::gds);
156  if (MI.getOperand(GDS).getImm())
157  return true;
158  }
159  return false;
160  }
161 }
162 
163 static bool isPermlane(const MachineInstr &MI) {
164  unsigned Opcode = MI.getOpcode();
165  return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
166  Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
167 }
168 
169 static bool isLdsDma(const MachineInstr &MI) {
170  return SIInstrInfo::isVALU(MI) &&
172 }
173 
174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
175  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
176  AMDGPU::OpName::simm16);
177  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
178 }
179 
182  MachineInstr *MI = SU->getInstr();
183  // If we are not in "HazardRecognizerMode" and therefore not being run from
184  // the scheduler, track possible stalls from hazards but don't insert noops.
185  auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
186 
187  if (MI->isBundle())
188  return NoHazard;
189 
190  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
191  return HazardType;
192 
193  if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
194  return HazardType;
195 
196  if (checkFPAtomicToDenormModeHazard(MI) > 0)
197  return HazardType;
198 
199  if (ST.hasNoDataDepHazard())
200  return NoHazard;
201 
202  // FIXME: Should flat be considered vmem?
203  if ((SIInstrInfo::isVMEM(*MI) ||
205  && checkVMEMHazards(MI) > 0)
206  return HazardType;
207 
208  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
209  return HazardType;
210 
211  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
212  return HazardType;
213 
214  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
215  return HazardType;
216 
217  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
218  return HazardType;
219 
222  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
223  return HazardType;
224 
225  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
226  return HazardType;
227 
228  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
229  return HazardType;
230 
231  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
232  return HazardType;
233 
234  if (((ST.hasReadM0MovRelInterpHazard() &&
235  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
237  (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
238  (ST.hasReadM0LdsDirectHazard() &&
239  MI->readsRegister(AMDGPU::LDS_DIRECT))) &&
240  checkReadM0Hazards(MI) > 0)
241  return HazardType;
242 
243  if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
244  return HazardType;
245 
246  if ((SIInstrInfo::isVMEM(*MI) ||
248  SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
249  return HazardType;
250 
251  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
252  return HazardType;
253 
254  return NoHazard;
255 }
256 
258  unsigned Quantity) {
259  while (Quantity > 0) {
260  unsigned Arg = std::min(Quantity, 8u);
261  Quantity -= Arg;
262  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
263  .addImm(Arg - 1);
264  }
265 }
266 
267 unsigned
268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
269  const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
270  assert(TSchedModel.getWriteProcResBegin(SC) !=
271  TSchedModel.getWriteProcResEnd(SC));
272  return TSchedModel.getWriteProcResBegin(SC)->Cycles;
273 }
274 
275 void GCNHazardRecognizer::processBundle() {
276  MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
278  // Check bundled MachineInstr's for hazards.
279  for (; MI != E && MI->isInsideBundle(); ++MI) {
280  CurrCycleInstr = &*MI;
281  unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
282 
283  if (IsHazardRecognizerMode) {
284  fixHazards(CurrCycleInstr);
285 
286  insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
287  }
288 
289  // It’s unnecessary to track more than MaxLookAhead instructions. Since we
290  // include the bundled MI directly after, only add a maximum of
291  // (MaxLookAhead - 1) noops to EmittedInstrs.
292  for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
293  EmittedInstrs.push_front(nullptr);
294 
295  EmittedInstrs.push_front(CurrCycleInstr);
296  EmittedInstrs.resize(MaxLookAhead);
297  }
298  CurrCycleInstr = nullptr;
299 }
300 
302  IsHazardRecognizerMode = true;
303  CurrCycleInstr = MI;
304  unsigned W = PreEmitNoopsCommon(MI);
305  fixHazards(MI);
306  CurrCycleInstr = nullptr;
307  return W;
308 }
309 
311  if (MI->isBundle())
312  return 0;
313 
314  int WaitStates = 0;
315 
316  if (SIInstrInfo::isSMRD(*MI))
317  return std::max(WaitStates, checkSMRDHazards(MI));
318 
319  if (ST.hasNSAtoVMEMBug())
320  WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
321 
322  WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
323 
324  if (ST.hasNoDataDepHazard())
325  return WaitStates;
326 
328  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
329 
330  if (SIInstrInfo::isVALU(*MI))
331  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
332 
333  if (SIInstrInfo::isDPP(*MI))
334  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
335 
336  if (isDivFMas(MI->getOpcode()))
337  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
338 
339  if (isRWLane(MI->getOpcode()))
340  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
341 
344  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
345  WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
346 
347  if (MI->isInlineAsm())
348  return std::max(WaitStates, checkInlineAsmHazards(MI));
349 
350  if (isSGetReg(MI->getOpcode()))
351  return std::max(WaitStates, checkGetRegHazards(MI));
352 
353  if (isSSetReg(MI->getOpcode()))
354  return std::max(WaitStates, checkSetRegHazards(MI));
355 
356  if (isRFE(MI->getOpcode()))
357  return std::max(WaitStates, checkRFEHazards(MI));
358 
359  if ((ST.hasReadM0MovRelInterpHazard() &&
360  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
362  (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
363  (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT)))
364  return std::max(WaitStates, checkReadM0Hazards(MI));
365 
366  if (SIInstrInfo::isMAI(*MI))
367  return std::max(WaitStates, checkMAIHazards(MI));
368 
369  if (SIInstrInfo::isVMEM(*MI) ||
372  return std::max(WaitStates, checkMAILdStHazards(MI));
373 
374  return WaitStates;
375 }
376 
378  EmittedInstrs.push_front(nullptr);
379 }
380 
382  // When the scheduler detects a stall, it will call AdvanceCycle() without
383  // emitting any instructions.
384  if (!CurrCycleInstr) {
385  EmittedInstrs.push_front(nullptr);
386  return;
387  }
388 
389  if (CurrCycleInstr->isBundle()) {
390  processBundle();
391  return;
392  }
393 
394  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
395  if (!NumWaitStates) {
396  CurrCycleInstr = nullptr;
397  return;
398  }
399 
400  // Keep track of emitted instructions
401  EmittedInstrs.push_front(CurrCycleInstr);
402 
403  // Add a nullptr for each additional wait state after the first. Make sure
404  // not to add more than getMaxLookAhead() items to the list, since we
405  // truncate the list to that size right after this loop.
406  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
407  i < e; ++i) {
408  EmittedInstrs.push_front(nullptr);
409  }
410 
411  // getMaxLookahead() is the largest number of wait states we will ever need
412  // to insert, so there is no point in keeping track of more than that many
413  // wait states.
414  EmittedInstrs.resize(getMaxLookAhead());
415 
416  CurrCycleInstr = nullptr;
417 }
418 
420  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
421 }
422 
423 //===----------------------------------------------------------------------===//
424 // Helper Functions
425 //===----------------------------------------------------------------------===//
426 
428 
429 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
430 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;
431 
432 // Search for a hazard in a block and its predecessors.
433 template <typename StateT>
434 static bool
435 hasHazard(StateT State,
436  function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
437  function_ref<void(StateT &, const MachineInstr &)> UpdateState,
438  const MachineBasicBlock *MBB,
441  for (auto E = MBB->instr_rend(); I != E; ++I) {
442  // No need to look at parent BUNDLE instructions.
443  if (I->isBundle())
444  continue;
445 
446  switch (IsHazard(State, *I)) {
447  case HazardFound:
448  return true;
449  case HazardExpired:
450  return false;
451  default:
452  // Continue search
453  break;
454  }
455 
456  if (I->isInlineAsm() || I->isMetaInstruction())
457  continue;
458 
459  UpdateState(State, *I);
460  }
461 
462  for (MachineBasicBlock *Pred : MBB->predecessors()) {
463  if (!Visited.insert(Pred).second)
464  continue;
465 
466  if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(),
467  Visited))
468  return true;
469  }
470 
471  return false;
472 }
473 
474 // Returns a minimum wait states since \p I walking all predecessors.
475 // Only scans until \p IsExpired does not return true.
476 // Can only be run in a hazard recognizer mode.
482  for (auto E = MBB->instr_rend(); I != E; ++I) {
483  // Don't add WaitStates for parent BUNDLE instructions.
484  if (I->isBundle())
485  continue;
486 
487  if (IsHazard(*I))
488  return WaitStates;
489 
490  if (I->isInlineAsm())
491  continue;
492 
493  WaitStates += GetNumWaitStates(*I);
494 
495  if (IsExpired(*I, WaitStates))
497  }
498 
499  int MinWaitStates = std::numeric_limits<int>::max();
500  for (MachineBasicBlock *Pred : MBB->predecessors()) {
501  if (!Visited.insert(Pred).second)
502  continue;
503 
504  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates,
505  IsExpired, Visited, GetNumWaitStates);
506 
507  MinWaitStates = std::min(MinWaitStates, W);
508  }
509 
510  return MinWaitStates;
511 }
512 
514  const MachineInstr *MI, IsExpiredFn IsExpired) {
516  return getWaitStatesSince(IsHazard, MI->getParent(),
517  std::next(MI->getReverseIterator()),
518  0, IsExpired, Visited);
519 }
520 
521 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
522  if (IsHazardRecognizerMode) {
523  auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
524  return WaitStates >= Limit;
525  };
526  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
527  }
528 
529  int WaitStates = 0;
530  for (MachineInstr *MI : EmittedInstrs) {
531  if (MI) {
532  if (IsHazard(*MI))
533  return WaitStates;
534 
535  if (MI->isInlineAsm())
536  continue;
537  }
538  ++WaitStates;
539 
540  if (WaitStates >= Limit)
541  break;
542  }
544 }
545 
546 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
547  IsHazardFn IsHazardDef,
548  int Limit) {
549  const SIRegisterInfo *TRI = ST.getRegisterInfo();
550 
551  auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
552  return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
553  };
554 
555  return getWaitStatesSince(IsHazardFn, Limit);
556 }
557 
558 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
559  int Limit) {
560  auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
561  return isSSetReg(MI.getOpcode()) && IsHazard(MI);
562  };
563 
564  return getWaitStatesSince(IsHazardFn, Limit);
565 }
566 
567 //===----------------------------------------------------------------------===//
568 // No-op Hazard Detection
569 //===----------------------------------------------------------------------===//
570 
571 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
572  MCRegister Reg) {
573  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
574  BV.set(*RUI);
575 }
576 
577 static void addRegsToSet(const SIRegisterInfo &TRI,
579  BitVector &Set) {
580  for (const MachineOperand &Op : Ops) {
581  if (Op.isReg())
582  addRegUnits(TRI, Set, Op.getReg().asMCReg());
583  }
584 }
585 
586 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
587  // XXX: Do we need to worry about implicit operands
588  addRegsToSet(TRI, MI.defs(), ClauseDefs);
589  addRegsToSet(TRI, MI.uses(), ClauseUses);
590 }
591 
593  return !SIInstrInfo::isSMRD(*MI);
594 }
595 
598 }
599 
600 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
601  // SMEM soft clause are only present on VI+, and only matter if xnack is
602  // enabled.
603  if (!ST.isXNACKEnabled())
604  return 0;
605 
606  bool IsSMRD = TII.isSMRD(*MEM);
607 
608  resetClause();
609 
610  // A soft-clause is any group of consecutive SMEM instructions. The
611  // instructions in this group may return out of order and/or may be
612  // replayed (i.e. the same instruction issued more than once).
613  //
614  // In order to handle these situations correctly we need to make sure that
615  // when a clause has more than one instruction, no instruction in the clause
616  // writes to a register that is read by another instruction in the clause
617  // (including itself). If we encounter this situation, we need to break the
618  // clause by inserting a non SMEM instruction.
619 
620  for (MachineInstr *MI : EmittedInstrs) {
621  // When we hit a non-SMEM instruction then we have passed the start of the
622  // clause and we can stop.
623  if (!MI)
624  break;
625 
627  break;
628 
629  addClauseInst(*MI);
630  }
631 
632  if (ClauseDefs.none())
633  return 0;
634 
635  // We need to make sure not to put loads and stores in the same clause if they
636  // use the same address. For now, just start a new clause whenever we see a
637  // store.
638  if (MEM->mayStore())
639  return 1;
640 
641  addClauseInst(*MEM);
642 
643  // If the set of defs and uses intersect then we cannot add this instruction
644  // to the clause, so we have a hazard.
645  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
646 }
647 
648 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
649  int WaitStatesNeeded = 0;
650 
651  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
652 
653  // This SMRD hazard only affects SI.
654  if (!ST.hasSMRDReadVALUDefHazard())
655  return WaitStatesNeeded;
656 
657  // A read of an SGPR by SMRD instruction requires 4 wait states when the
658  // SGPR was written by a VALU instruction.
659  int SmrdSgprWaitStates = 4;
660  auto IsHazardDefFn = [this](const MachineInstr &MI) {
661  return TII.isVALU(MI);
662  };
663  auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
664  return TII.isSALU(MI);
665  };
666 
667  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
668 
669  for (const MachineOperand &Use : SMRD->uses()) {
670  if (!Use.isReg())
671  continue;
672  int WaitStatesNeededForUse =
673  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
674  SmrdSgprWaitStates);
675  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
676 
677  // This fixes what appears to be undocumented hardware behavior in SI where
678  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
679  // needs some number of nops in between. We don't know how many we need, but
680  // let's use 4. This wasn't discovered before probably because the only
681  // case when this happens is when we expand a 64-bit pointer into a full
682  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
683  // probably never encountered in the closed-source land.
684  if (IsBufferSMRD) {
685  int WaitStatesNeededForUse =
686  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
687  IsBufferHazardDefFn,
688  SmrdSgprWaitStates);
689  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
690  }
691  }
692 
693  return WaitStatesNeeded;
694 }
695 
696 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
698  return 0;
699 
700  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
701 
702  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
703  // SGPR was written by a VALU Instruction.
704  const int VmemSgprWaitStates = 5;
705  auto IsHazardDefFn = [this](const MachineInstr &MI) {
706  return TII.isVALU(MI);
707  };
708  for (const MachineOperand &Use : VMEM->uses()) {
709  if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
710  continue;
711 
712  int WaitStatesNeededForUse =
713  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
714  VmemSgprWaitStates);
715  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
716  }
717  return WaitStatesNeeded;
718 }
719 
720 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
721  const SIRegisterInfo *TRI = ST.getRegisterInfo();
722  const SIInstrInfo *TII = ST.getInstrInfo();
723 
724  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
725  int DppVgprWaitStates = 2;
726  int DppExecWaitStates = 5;
727  int WaitStatesNeeded = 0;
728  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
729  return TII->isVALU(MI);
730  };
731 
732  for (const MachineOperand &Use : DPP->uses()) {
733  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
734  continue;
735  int WaitStatesNeededForUse =
736  DppVgprWaitStates - getWaitStatesSinceDef(
737  Use.getReg(),
738  [](const MachineInstr &) { return true; },
739  DppVgprWaitStates);
740  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
741  }
742 
743  WaitStatesNeeded = std::max(
744  WaitStatesNeeded,
745  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
746  DppExecWaitStates));
747 
748  return WaitStatesNeeded;
749 }
750 
751 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
752  const SIInstrInfo *TII = ST.getInstrInfo();
753 
754  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
755  // instruction.
756  const int DivFMasWaitStates = 4;
757  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
758  return TII->isVALU(MI);
759  };
760  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
761  DivFMasWaitStates);
762 
763  return DivFMasWaitStates - WaitStatesNeeded;
764 }
765 
766 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
767  const SIInstrInfo *TII = ST.getInstrInfo();
768  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
769 
770  const int GetRegWaitStates = 2;
771  auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
772  return GetRegHWReg == getHWReg(TII, MI);
773  };
774  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
775 
776  return GetRegWaitStates - WaitStatesNeeded;
777 }
778 
779 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
780  const SIInstrInfo *TII = ST.getInstrInfo();
781  unsigned HWReg = getHWReg(TII, *SetRegInstr);
782 
783  const int SetRegWaitStates = ST.getSetRegWaitStates();
784  auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
785  return HWReg == getHWReg(TII, MI);
786  };
787  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
788  return SetRegWaitStates - WaitStatesNeeded;
789 }
790 
791 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
792  if (!MI.mayStore())
793  return -1;
794 
795  const SIInstrInfo *TII = ST.getInstrInfo();
796  unsigned Opcode = MI.getOpcode();
797  const MCInstrDesc &Desc = MI.getDesc();
798 
799  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
800  int VDataRCID = -1;
801  if (VDataIdx != -1)
802  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
803 
804  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
805  // There is no hazard if the instruction does not use vector regs
806  // (like wbinvl1)
807  if (VDataIdx == -1)
808  return -1;
809  // For MUBUF/MTBUF instructions this hazard only exists if the
810  // instruction is not using a register in the soffset field.
811  const MachineOperand *SOffset =
812  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
813  // If we have no soffset operand, then assume this field has been
814  // hardcoded to zero.
815  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
816  (!SOffset || !SOffset->isReg()))
817  return VDataIdx;
818  }
819 
820  // MIMG instructions create a hazard if they don't use a 256-bit T# and
821  // the store size is greater than 8 bytes and they have more than two bits
822  // of their dmask set.
823  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
824  if (TII->isMIMG(MI)) {
825  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
826  assert(SRsrcIdx != -1 &&
827  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
828  (void)SRsrcIdx;
829  }
830 
831  if (TII->isFLAT(MI)) {
832  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
833  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
834  return DataIdx;
835  }
836 
837  return -1;
838 }
839 
840 int
841 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
842  const MachineRegisterInfo &MRI) {
843  // Helper to check for the hazard where VMEM instructions that store more than
844  // 8 bytes can have there store data over written by the next instruction.
845  const SIRegisterInfo *TRI = ST.getRegisterInfo();
846 
847  const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1;
848  int WaitStatesNeeded = 0;
849 
850  if (!TRI->isVectorRegister(MRI, Def.getReg()))
851  return WaitStatesNeeded;
852  Register Reg = Def.getReg();
853  auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
854  int DataIdx = createsVALUHazard(MI);
855  return DataIdx >= 0 &&
856  TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
857  };
858  int WaitStatesNeededForDef =
859  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
860  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
861 
862  return WaitStatesNeeded;
863 }
864 
865 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
866  int WaitStatesNeeded = 0;
867 
869  const int TransDefWaitstates = 1;
870 
871  auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
872  if (!SIInstrInfo::isTRANS(MI))
873  return false;
874  const SIRegisterInfo *TRI = ST.getRegisterInfo();
875  const SIInstrInfo *TII = ST.getInstrInfo();
876  Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
877 
878  for (const MachineOperand &Use : VALU->explicit_uses()) {
879  if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
880  return true;
881  }
882 
883  return false;
884  };
885 
886  int WaitStatesNeededForDef =
887  TransDefWaitstates -
888  getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
889  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
890  }
891 
892  if (ST.hasDstSelForwardingHazard()) {
893  const int Shift16DefWaitstates = 1;
894 
895  auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) {
896  if (!SIInstrInfo::isVALU(MI))
897  return false;
898  const SIInstrInfo *TII = ST.getInstrInfo();
899  if (SIInstrInfo::isSDWA(MI)) {
900  if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
901  if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
902  return false;
903  } else {
904  if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(),
905  AMDGPU::OpName::op_sel) == -1) ||
906  !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)
907  ->getImm() &
909  return false;
910  }
911  const SIRegisterInfo *TRI = ST.getRegisterInfo();
912  if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
913  Register Def = Dst->getReg();
914 
915  for (const MachineOperand &Use : VALU->explicit_uses()) {
916  if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
917  return true;
918  }
919  }
920 
921  return false;
922  };
923 
924  int WaitStatesNeededForDef =
925  Shift16DefWaitstates -
926  getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
927  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
928  }
929 
930  if (ST.hasVDecCoExecHazard()) {
931  const int VALUWriteSGPRVALUReadWaitstates = 2;
932  const int VALUWriteEXECRWLane = 4;
933  const int VALUWriteVGPRReadlaneRead = 1;
934 
935  const SIRegisterInfo *TRI = ST.getRegisterInfo();
936  const MachineRegisterInfo &MRI = MF.getRegInfo();
938  auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
939  if (!SIInstrInfo::isVALU(MI))
940  return false;
941  return MI.modifiesRegister(UseReg, TRI);
942  };
943 
944  for (const MachineOperand &Use : VALU->explicit_uses()) {
945  if (!Use.isReg())
946  continue;
947 
948  UseReg = Use.getReg();
949  if (TRI->isSGPRReg(MRI, UseReg)) {
950  int WaitStatesNeededForDef =
951  VALUWriteSGPRVALUReadWaitstates -
952  getWaitStatesSince(IsVALUDefSGPRFn,
953  VALUWriteSGPRVALUReadWaitstates);
954  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
955  }
956  }
957 
958  if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
959  UseReg = AMDGPU::VCC;
960  int WaitStatesNeededForDef =
961  VALUWriteSGPRVALUReadWaitstates -
962  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
963  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
964  }
965 
966  switch (VALU->getOpcode()) {
967  case AMDGPU::V_READLANE_B32:
968  case AMDGPU::V_READFIRSTLANE_B32: {
969  MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
970  UseReg = Src->getReg();
971  int WaitStatesNeededForDef =
972  VALUWriteVGPRReadlaneRead -
973  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
974  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
975  }
977  case AMDGPU::V_WRITELANE_B32: {
978  UseReg = AMDGPU::EXEC;
979  int WaitStatesNeededForDef =
980  VALUWriteEXECRWLane -
981  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
982  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
983  break;
984  }
985  default:
986  break;
987  }
988  }
989 
990  // This checks for the hazard where VMEM instructions that store more than
991  // 8 bytes can have there store data over written by the next instruction.
992  if (!ST.has12DWordStoreHazard())
993  return WaitStatesNeeded;
994 
995  const MachineRegisterInfo &MRI = MF.getRegInfo();
996 
997  for (const MachineOperand &Def : VALU->defs()) {
998  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
999  }
1000 
1001  return WaitStatesNeeded;
1002 }
1003 
1004 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
1005  // This checks for hazards associated with inline asm statements.
1006  // Since inline asms can contain just about anything, we use this
1007  // to call/leverage other check*Hazard routines. Note that
1008  // this function doesn't attempt to address all possible inline asm
1009  // hazards (good luck), but is a collection of what has been
1010  // problematic thus far.
1011 
1012  // see checkVALUHazards()
1013  if (!ST.has12DWordStoreHazard())
1014  return 0;
1015 
1016  const MachineRegisterInfo &MRI = MF.getRegInfo();
1017  int WaitStatesNeeded = 0;
1018 
1019  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
1020  I != E; ++I) {
1021  const MachineOperand &Op = IA->getOperand(I);
1022  if (Op.isReg() && Op.isDef()) {
1023  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
1024  }
1025  }
1026 
1027  return WaitStatesNeeded;
1028 }
1029 
1030 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
1031  const SIInstrInfo *TII = ST.getInstrInfo();
1032  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1033  const MachineRegisterInfo &MRI = MF.getRegInfo();
1034 
1035  const MachineOperand *LaneSelectOp =
1036  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
1037 
1038  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
1039  return 0;
1040 
1041  Register LaneSelectReg = LaneSelectOp->getReg();
1042  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
1043 
1044  const int RWLaneWaitStates = 4;
1045  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1046  RWLaneWaitStates);
1047  return RWLaneWaitStates - WaitStatesSince;
1048 }
1049 
1050 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
1051  if (!ST.hasRFEHazards())
1052  return 0;
1053 
1054  const SIInstrInfo *TII = ST.getInstrInfo();
1055 
1056  const int RFEWaitStates = 1;
1057 
1058  auto IsHazardFn = [TII](const MachineInstr &MI) {
1059  return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1060  };
1061  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1062  return RFEWaitStates - WaitStatesNeeded;
1063 }
1064 
1065 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
1066  const SIInstrInfo *TII = ST.getInstrInfo();
1067  const int ReadM0WaitStates = 1;
1068  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1069  return ReadM0WaitStates -
1070  getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1071 }
1072 
1073 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1074  fixVMEMtoScalarWriteHazards(MI);
1075  fixVcmpxPermlaneHazards(MI);
1076  fixSMEMtoVectorWriteHazards(MI);
1077  fixVcmpxExecWARHazard(MI);
1078  fixLdsBranchVmemWARHazard(MI);
1079  if (ST.hasLdsDirect()) {
1080  fixLdsDirectVALUHazard(MI);
1081  fixLdsDirectVMEMHazard(MI);
1082  }
1083  fixVALUPartialForwardingHazard(MI);
1084  fixVALUTransUseHazard(MI);
1085  fixWMMAHazards(MI);
1086 }
1087 
1088 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1089  if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1090  return false;
1091 
1092  const SIInstrInfo *TII = ST.getInstrInfo();
1093  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1094  auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1095  return (TII->isVOPC(MI) ||
1096  ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) &&
1097  MI.modifiesRegister(AMDGPU::EXEC, TRI);
1098  };
1099 
1100  auto IsExpiredFn = [](const MachineInstr &MI, int) {
1101  unsigned Opc = MI.getOpcode();
1102  return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
1103  Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
1104  };
1105 
1106  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1108  return false;
1109 
1110  // V_NOP will be discarded by SQ.
1111  // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1112  // which is always a VGPR and available.
1113  auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1114  Register Reg = Src0->getReg();
1115  bool IsUndef = Src0->isUndef();
1116  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1117  TII->get(AMDGPU::V_MOV_B32_e32))
1118  .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
1119  .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
1120 
1121  return true;
1122 }
1123 
1124 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1125  if (!ST.hasVMEMtoScalarWriteHazard())
1126  return false;
1127 
1129  return false;
1130 
1131  if (MI->getNumDefs() == 0)
1132  return false;
1133 
1134  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1135 
1136  auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1139  return false;
1140 
1141  for (const MachineOperand &Def : MI->defs()) {
1142  const MachineOperand *Op =
1143  I.findRegisterUseOperand(Def.getReg(), false, TRI);
1144  if (!Op)
1145  continue;
1146  return true;
1147  }
1148  return false;
1149  };
1150 
1151  auto IsExpiredFn = [](const MachineInstr &MI, int) {
1152  return SIInstrInfo::isVALU(MI) ||
1153  (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1154  !MI.getOperand(0).getImm()) ||
1155  (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1156  MI.getOperand(0).getImm() == 0xffe3);
1157  };
1158 
1159  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1161  return false;
1162 
1163  const SIInstrInfo *TII = ST.getInstrInfo();
1164  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1165  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1166  .addImm(0xffe3);
1167  return true;
1168 }
1169 
1170 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1171  if (!ST.hasSMEMtoVectorWriteHazard())
1172  return false;
1173 
1174  if (!SIInstrInfo::isVALU(*MI))
1175  return false;
1176 
1177  unsigned SDSTName;
1178  switch (MI->getOpcode()) {
1179  case AMDGPU::V_READLANE_B32:
1180  case AMDGPU::V_READFIRSTLANE_B32:
1181  SDSTName = AMDGPU::OpName::vdst;
1182  break;
1183  default:
1184  SDSTName = AMDGPU::OpName::sdst;
1185  break;
1186  }
1187 
1188  const SIInstrInfo *TII = ST.getInstrInfo();
1189  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1190  const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
1191  const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
1192  if (!SDST) {
1193  for (const auto &MO : MI->implicit_operands()) {
1194  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
1195  SDST = &MO;
1196  break;
1197  }
1198  }
1199  }
1200 
1201  if (!SDST)
1202  return false;
1203 
1204  const Register SDSTReg = SDST->getReg();
1205  auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1206  return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
1207  };
1208 
1209  auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1210  if (TII->isSALU(MI)) {
1211  switch (MI.getOpcode()) {
1212  case AMDGPU::S_SETVSKIP:
1213  case AMDGPU::S_VERSION:
1214  case AMDGPU::S_WAITCNT_VSCNT:
1215  case AMDGPU::S_WAITCNT_VMCNT:
1216  case AMDGPU::S_WAITCNT_EXPCNT:
1217  // These instructions cannot not mitigate the hazard.
1218  return false;
1219  case AMDGPU::S_WAITCNT_LGKMCNT:
1220  // Reducing lgkmcnt count to 0 always mitigates the hazard.
1221  return (MI.getOperand(1).getImm() == 0) &&
1222  (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1223  case AMDGPU::S_WAITCNT: {
1224  const int64_t Imm = MI.getOperand(0).getImm();
1225  AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1226  return (Decoded.LgkmCnt == 0);
1227  }
1228  default:
1229  // SOPP instructions cannot mitigate the hazard.
1230  if (TII->isSOPP(MI))
1231  return false;
1232  // At this point the SALU can be assumed to mitigate the hazard
1233  // because either:
1234  // (a) it is independent of the at risk SMEM (breaking chain),
1235  // or
1236  // (b) it is dependent on the SMEM, in which case an appropriate
1237  // s_waitcnt lgkmcnt _must_ exist between it and the at risk
1238  // SMEM instruction.
1239  return true;
1240  }
1241  }
1242  return false;
1243  };
1244 
1245  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1247  return false;
1248 
1249  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1250  TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1251  .addImm(0);
1252  return true;
1253 }
1254 
1255 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1257  return false;
1258 
1259  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1260  if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1261  return false;
1262 
1263  auto IsHazardFn = [TRI](const MachineInstr &I) {
1264  if (SIInstrInfo::isVALU(I))
1265  return false;
1266  return I.readsRegister(AMDGPU::EXEC, TRI);
1267  };
1268 
1269  const SIInstrInfo *TII = ST.getInstrInfo();
1270  auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1271  if (SIInstrInfo::isVALU(MI)) {
1272  if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1273  return true;
1274  for (auto MO : MI.implicit_operands())
1275  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1276  return true;
1277  }
1278  if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1279  (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe)
1280  return true;
1281  return false;
1282  };
1283 
1284  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1286  return false;
1287 
1288  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1289  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1290  .addImm(0xfffe);
1291  return true;
1292 }
1293 
1295  const GCNSubtarget &ST) {
1296  if (!ST.hasLdsBranchVmemWARHazard())
1297  return false;
1298 
1299  // Check if the necessary condition for the hazard is met: both LDS and VMEM
1300  // instructions need to appear in the same function.
1301  bool HasLds = false;
1302  bool HasVmem = false;
1303  for (auto &MBB : MF) {
1304  for (auto &MI : MBB) {
1305  HasLds |= SIInstrInfo::isDS(MI);
1306  HasVmem |=
1308  if (HasLds && HasVmem)
1309  return true;
1310  }
1311  }
1312  return false;
1313 }
1314 
1315 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1316  if (!RunLdsBranchVmemWARHazardFixup)
1317  return false;
1318 
1320 
1321  auto IsHazardInst = [](const MachineInstr &MI) {
1322  if (SIInstrInfo::isDS(MI))
1323  return 1;
1325  return 2;
1326  return 0;
1327  };
1328 
1329  auto InstType = IsHazardInst(*MI);
1330  if (!InstType)
1331  return false;
1332 
1333  auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1334  return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1335  I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1336  !I.getOperand(1).getImm());
1337  };
1338 
1339  auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1340  if (!I.isBranch())
1341  return false;
1342 
1343  auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1344  auto InstType2 = IsHazardInst(I);
1345  return InstType2 && InstType != InstType2;
1346  };
1347 
1348  auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1349  auto InstType2 = IsHazardInst(I);
1350  if (InstType == InstType2)
1351  return true;
1352 
1353  return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1354  I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1355  !I.getOperand(1).getImm();
1356  };
1357 
1360  };
1361 
1362  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1364  return false;
1365 
1366  const SIInstrInfo *TII = ST.getInstrInfo();
1367  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1368  TII->get(AMDGPU::S_WAITCNT_VSCNT))
1369  .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1370  .addImm(0);
1371 
1372  return true;
1373 }
1374 
1375 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
1376  if (!SIInstrInfo::isLDSDIR(*MI))
1377  return false;
1378 
1379  const int NoHazardWaitStates = 15;
1380  const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1381  const Register VDSTReg = VDST->getReg();
1382 
1383  bool VisitedTrans = false;
1384  auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
1385  if (!SIInstrInfo::isVALU(I))
1386  return false;
1387  VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
1388  // Cover both WAR and WAW
1389  return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1390  };
1391  auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
1392  if (WaitStates >= NoHazardWaitStates)
1393  return true;
1394  // Instructions which cause va_vdst==0 expire hazard
1397  };
1398  auto GetWaitStatesFn = [](const MachineInstr &MI) {
1399  return SIInstrInfo::isVALU(MI) ? 1 : 0;
1400  };
1401 
1403  auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
1404  std::next(MI->getReverseIterator()), 0,
1405  IsExpiredFn, Visited, GetWaitStatesFn);
1406 
1407  // Transcendentals can execute in parallel to other VALUs.
1408  // This makes va_vdst count unusable with a mixture of VALU and TRANS.
1409  if (VisitedTrans)
1410  Count = 0;
1411 
1412  MachineOperand *WaitVdstOp =
1413  TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst);
1414  WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates));
1415 
1416  return true;
1417 }
1418 
1419 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
1420  if (!SIInstrInfo::isLDSDIR(*MI))
1421  return false;
1422 
1423  const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1424  const Register VDSTReg = VDST->getReg();
1425 
1426  auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
1428  !SIInstrInfo::isDS(I))
1429  return false;
1430  return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1431  };
1432  auto IsExpiredFn = [](const MachineInstr &I, int) {
1433  return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) ||
1434  (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
1435  (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1436  I.getOperand(0).getImm() == 0xffe3);
1437  };
1438 
1439  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1441  return false;
1442 
1443  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1444  TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1445  .addImm(0xffe3);
1446 
1447  return true;
1448 }
1449 
1450 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
1451  if (!ST.isWave64())
1452  return false;
1454  return false;
1455  if (!SIInstrInfo::isVALU(*MI))
1456  return false;
1457 
1458  SmallSetVector<Register, 4> SrcVGPRs;
1459 
1460  for (const MachineOperand &Use : MI->explicit_uses()) {
1461  if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1462  SrcVGPRs.insert(Use.getReg());
1463  }
1464 
1465  // Only applies with >= 2 unique VGPR sources
1466  if (SrcVGPRs.size() <= 1)
1467  return false;
1468 
1469  // Look for the following pattern:
1470  // Va <- VALU [PreExecPos]
1471  // intv1
1472  // Exec <- SALU [ExecPos]
1473  // intv2
1474  // Vb <- VALU [PostExecPos]
1475  // intv3
1476  // MI Va, Vb (WaitState = 0)
1477  //
1478  // Where:
1479  // intv1 + intv2 <= 2 VALUs
1480  // intv3 <= 4 VALUs
1481  //
1482  // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1483 
1484  const int Intv1plus2MaxVALUs = 2;
1485  const int Intv3MaxVALUs = 4;
1486  const int IntvMaxVALUs = 6;
1487  const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
1488 
1489  struct StateType {
1491  int ExecPos = std::numeric_limits<int>::max();
1492  int VALUs = 0;
1493  };
1494 
1495  StateType State;
1496 
1497  // This overloads expiry testing with all the hazard detection
1498  auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1499  // Too many VALU states have passed
1500  if (State.VALUs > NoHazardVALUWaitStates)
1501  return HazardExpired;
1502 
1503  // Instructions which cause va_vdst==0 expire hazard
1506  (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1507  I.getOperand(0).getImm() == 0x0fff))
1508  return HazardExpired;
1509 
1510  // Track registers writes
1511  bool Changed = false;
1512  if (SIInstrInfo::isVALU(I)) {
1513  for (Register Src : SrcVGPRs) {
1514  if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
1515  State.DefPos[Src] = State.VALUs;
1516  Changed = true;
1517  }
1518  }
1519  } else if (SIInstrInfo::isSALU(I)) {
1520  if (State.ExecPos == std::numeric_limits<int>::max()) {
1521  if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
1522  State.ExecPos = State.VALUs;
1523  Changed = true;
1524  }
1525  }
1526  }
1527 
1528  // Early expiration: too many VALUs in intv3
1529  if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
1530  return HazardExpired;
1531 
1532  // Only evaluate state if something changed
1533  if (!Changed)
1534  return NoHazardFound;
1535 
1536  // Determine positions of VALUs pre/post exec change
1537  if (State.ExecPos == std::numeric_limits<int>::max())
1538  return NoHazardFound;
1539 
1540  int PreExecPos = std::numeric_limits<int>::max();
1541  int PostExecPos = std::numeric_limits<int>::max();
1542 
1543  for (auto Entry : State.DefPos) {
1544  int DefVALUs = Entry.second;
1545  if (DefVALUs != std::numeric_limits<int>::max()) {
1546  if (DefVALUs >= State.ExecPos)
1547  PreExecPos = std::min(PreExecPos, DefVALUs);
1548  else if (DefVALUs < State.ExecPos)
1549  PostExecPos = std::min(PostExecPos, DefVALUs);
1550  }
1551  }
1552 
1553  // Need a VALUs post exec change
1554  if (PostExecPos == std::numeric_limits<int>::max())
1555  return NoHazardFound;
1556 
1557  // Too many VALUs in intv3?
1558  int Intv3VALUs = PostExecPos;
1559  if (Intv3VALUs > Intv3MaxVALUs)
1560  return HazardExpired;
1561 
1562  // Too many VALUs in intv2?
1563  int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
1564  if (Intv2VALUs > Intv1plus2MaxVALUs)
1565  return HazardExpired;
1566 
1567  // Need a VALUs pre exec change
1568  if (PreExecPos == std::numeric_limits<int>::max())
1569  return NoHazardFound;
1570 
1571  // Too many VALUs in intv1?
1572  int Intv1VALUs = PreExecPos - State.ExecPos;
1573  if (Intv1VALUs > Intv1plus2MaxVALUs)
1574  return HazardExpired;
1575 
1576  // Too many VALUs in intv1 + intv2
1577  if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
1578  return HazardExpired;
1579 
1580  return HazardFound;
1581  };
1582  auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1583  if (SIInstrInfo::isVALU(MI))
1584  State.VALUs += 1;
1585  };
1586 
1588  if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1589  std::next(MI->getReverseIterator()), Visited))
1590  return false;
1591 
1592  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1593  TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1594  .addImm(0x0fff);
1595 
1596  return true;
1597 }
1598 
1599 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
1600  if (!ST.hasVALUTransUseHazard())
1601  return false;
1602  if (!SIInstrInfo::isVALU(*MI))
1603  return false;
1604 
1605  SmallSet<Register, 4> SrcVGPRs;
1606 
1607  for (const MachineOperand &Use : MI->explicit_uses()) {
1608  if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1609  SrcVGPRs.insert(Use.getReg());
1610  }
1611 
1612  // Look for the following pattern:
1613  // Va <- TRANS VALU
1614  // intv
1615  // MI Va (WaitState = 0)
1616  //
1617  // Where:
1618  // intv <= 5 VALUs / 1 TRANS
1619  //
1620  // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1621 
1622  const int IntvMaxVALUs = 5;
1623  const int IntvMaxTRANS = 1;
1624 
1625  struct StateType {
1626  int VALUs = 0;
1627  int TRANS = 0;
1628  };
1629 
1630  StateType State;
1631 
1632  // This overloads expiry testing with all the hazard detection
1633  auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1634  // Too many VALU states have passed
1635  if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
1636  return HazardExpired;
1637 
1638  // Instructions which cause va_vdst==0 expire hazard
1641  (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1642  I.getOperand(0).getImm() == 0x0fff))
1643  return HazardExpired;
1644 
1645  // Track registers writes
1646  if (SIInstrInfo::isTRANS(I)) {
1647  for (Register Src : SrcVGPRs) {
1648  if (I.modifiesRegister(Src, &TRI)) {
1649  return HazardFound;
1650  }
1651  }
1652  }
1653 
1654  return NoHazardFound;
1655  };
1656  auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1657  if (SIInstrInfo::isVALU(MI))
1658  State.VALUs += 1;
1660  State.TRANS += 1;
1661  };
1662 
1664  if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1665  std::next(MI->getReverseIterator()), Visited))
1666  return false;
1667 
1668  // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
1669  // avoided (mask 0x0fff achieves this).
1670  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1671  TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1672  .addImm(0x0fff);
1673 
1674  return true;
1675 }
1676 
1677 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
1678  if (!SIInstrInfo::isWMMA(*MI))
1679  return false;
1680 
1681  const SIInstrInfo *TII = ST.getInstrInfo();
1682  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1683 
1684  auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) {
1685  if (!SIInstrInfo::isWMMA(I))
1686  return false;
1687 
1688  // Src0 or Src1 of the current wmma instruction overlaps with the dest of
1689  // the previous wmma.
1690  const Register CurSrc0Reg =
1691  TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg();
1692  const Register CurSrc1Reg =
1693  TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
1694 
1695  const Register PrevDstReg =
1696  TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
1697 
1698  if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
1699  TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
1700  return true;
1701  }
1702 
1703  // Src2 of the current wmma instruction overlaps with the dest of the
1704  // previous wmma.
1705  const MachineOperand *Src2 =
1706  TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
1707  const Register CurSrc2Reg = Src2->isReg() ? Src2->getReg() : Register();
1708 
1709  if (CurSrc2Reg != AMDGPU::NoRegister &&
1710  TRI->regsOverlap(PrevDstReg, CurSrc2Reg)) {
1711 
1712  const MachineOperand *Src2Mods =
1713  TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers);
1714  const bool NoSrc2Mods =
1715  (Src2Mods->getImm() & (SISrcMods::NEG | SISrcMods::NEG_HI)) == 0;
1716  // Exception: there is no hazard if the wmma instructions are of the same
1717  // type and there is no input modifier on src2 of the current instruction.
1718  return !(NoSrc2Mods && (TII->pseudoToMCOpcode(I.getOpcode()) ==
1719  TII->pseudoToMCOpcode(MI->getOpcode())));
1720  }
1721 
1722  return false;
1723  };
1724 
1725  auto IsExpiredFn = [](const MachineInstr &I, int) {
1726  return SIInstrInfo::isVALU(I);
1727  };
1728 
1729  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1731  return false;
1732 
1733  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
1734 
1735  return true;
1736 }
1737 
1738 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1739  int NSAtoVMEMWaitStates = 1;
1740 
1741  if (!ST.hasNSAtoVMEMBug())
1742  return 0;
1743 
1745  return 0;
1746 
1747  const SIInstrInfo *TII = ST.getInstrInfo();
1748  const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1749  if (!Offset || (Offset->getImm() & 6) == 0)
1750  return 0;
1751 
1752  auto IsHazardFn = [TII](const MachineInstr &I) {
1753  if (!SIInstrInfo::isMIMG(I))
1754  return false;
1755  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1756  return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1757  TII->getInstSizeInBytes(I) >= 16;
1758  };
1759 
1760  return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1761 }
1762 
1763 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1764  int FPAtomicToDenormModeWaitStates = 3;
1765 
1766  if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1767  return 0;
1768 
1769  auto IsHazardFn = [](const MachineInstr &I) {
1771  return false;
1772  return SIInstrInfo::isFPAtomic(I);
1773  };
1774 
1775  auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1776  if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1777  return true;
1778 
1779  switch (MI.getOpcode()) {
1780  case AMDGPU::S_WAITCNT:
1781  case AMDGPU::S_WAITCNT_VSCNT:
1782  case AMDGPU::S_WAITCNT_VMCNT:
1783  case AMDGPU::S_WAITCNT_EXPCNT:
1784  case AMDGPU::S_WAITCNT_LGKMCNT:
1785  case AMDGPU::S_WAIT_IDLE:
1786  return true;
1787  default:
1788  break;
1789  }
1790 
1791  return false;
1792  };
1793 
1794  return FPAtomicToDenormModeWaitStates -
1795  ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1796 }
1797 
1798 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1800 
1801  return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1802 }
1803 
1804 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) {
1805  // Early exit if no padding is requested.
1806  if (MFMAPaddingRatio == 0)
1807  return 0;
1808 
1810  if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
1811  return 0;
1812 
1813  int NeighborMFMALatency = 0;
1814  auto IsNeighboringMFMA = [&NeighborMFMALatency,
1815  this](const MachineInstr &MI) {
1816  if (!SIInstrInfo::isMFMA(MI))
1817  return false;
1818 
1819  NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
1820  return true;
1821  };
1822 
1823  const int MaxMFMAPipelineWaitStates = 16;
1824  int WaitStatesSinceNeighborMFMA =
1825  getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
1826 
1827  int NeighborMFMAPaddingNeeded =
1828  (NeighborMFMALatency * MFMAPaddingRatio / 100) -
1829  WaitStatesSinceNeighborMFMA;
1830 
1831  return std::max(0, NeighborMFMAPaddingNeeded);
1832 }
1833 
1834 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1835  int WaitStatesNeeded = 0;
1836  unsigned Opc = MI->getOpcode();
1837 
1838  auto IsVALUFn = [](const MachineInstr &MI) {
1839  return SIInstrInfo::isVALU(MI);
1840  };
1841 
1842  if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1843  const int LegacyVALUWritesVGPRWaitStates = 2;
1844  const int VALUWritesExecWaitStates = 4;
1845  const int MaxWaitStates = 4;
1846 
1847  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1848  getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1849  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1850 
1851  if (WaitStatesNeeded < MaxWaitStates) {
1852  for (const MachineOperand &Use : MI->explicit_uses()) {
1853  const int MaxWaitStates = 2;
1854 
1855  if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1856  continue;
1857 
1858  int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1859  getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1860  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1861 
1862  if (WaitStatesNeeded == MaxWaitStates)
1863  break;
1864  }
1865  }
1866  }
1867 
1868  for (const MachineOperand &Op : MI->explicit_operands()) {
1869  if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1870  continue;
1871 
1872  if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1873  continue;
1874 
1875  const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1876  const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1877  const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1878  const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1879  const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1880  const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1881  const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1882  const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1883  const int MaxWaitStates = 18;
1884  Register Reg = Op.getReg();
1885  unsigned HazardDefLatency = 0;
1886 
1887  auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
1888  this](const MachineInstr &MI) {
1889  if (!SIInstrInfo::isMFMA(MI))
1890  return false;
1891  Register DstReg = MI.getOperand(0).getReg();
1892  if (DstReg == Reg)
1893  return false;
1894  HazardDefLatency =
1895  std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1896  return TRI.regsOverlap(DstReg, Reg);
1897  };
1898 
1899  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1900  MaxWaitStates);
1901  int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1902  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1903  int OpNo = MI->getOperandNo(&Op);
1904  if (OpNo == SrcCIdx) {
1905  NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1906  } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1907  switch (HazardDefLatency) {
1908  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1909  break;
1910  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1911  break;
1912  case 16: LLVM_FALLTHROUGH;
1913  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1914  break;
1915  }
1916  } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1917  switch (HazardDefLatency) {
1918  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1919  break;
1920  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1921  break;
1922  case 16: LLVM_FALLTHROUGH;
1923  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1924  break;
1925  }
1926  }
1927 
1928  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1929  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1930 
1931  if (WaitStatesNeeded == MaxWaitStates)
1932  return WaitStatesNeeded; // Early exit.
1933 
1934  auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
1935  if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1936  return false;
1937  Register DstReg = MI.getOperand(0).getReg();
1938  return TRI.regsOverlap(Reg, DstReg);
1939  };
1940 
1941  const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1942  const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1943  const int AccVGPRWriteAccVgprReadWaitStates = 3;
1944  NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1945  if (OpNo == SrcCIdx)
1946  NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1947  else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1948  NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1949 
1950  WaitStatesNeededForUse = NeedWaitStates -
1951  getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1952  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1953 
1954  if (WaitStatesNeeded == MaxWaitStates)
1955  return WaitStatesNeeded; // Early exit.
1956  }
1957 
1958  if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1959  const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1960  const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1961  const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1962  const int MaxWaitStates = 13;
1963  Register DstReg = MI->getOperand(0).getReg();
1964  unsigned HazardDefLatency = 0;
1965 
1966  auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
1967  this](const MachineInstr &MI) {
1968  if (!SIInstrInfo::isMFMA(MI))
1969  return false;
1970  Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
1971  HazardDefLatency =
1972  std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1973  return TRI.regsOverlap(Reg, DstReg);
1974  };
1975 
1976  int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1977  int NeedWaitStates;
1978  switch (HazardDefLatency) {
1979  case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1980  break;
1981  case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1982  break;
1983  case 16: LLVM_FALLTHROUGH;
1984  default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1985  break;
1986  }
1987 
1988  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1989  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1990  }
1991 
1992  // Pad neighboring MFMA with noops for better inter-wave performance.
1993  WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
1994 
1995  return WaitStatesNeeded;
1996 }
1997 
1998 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1999  int WaitStatesNeeded = 0;
2000  unsigned Opc = MI->getOpcode();
2001 
2002  auto IsLegacyVALUFn = [](const MachineInstr &MI) {
2004  };
2005 
2006  auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
2009  };
2010 
2011  if (!SIInstrInfo::isMFMA(*MI))
2012  return WaitStatesNeeded;
2013 
2014  const int VALUWritesExecWaitStates = 4;
2015  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
2016  getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
2017  VALUWritesExecWaitStates);
2018  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2019 
2020  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
2021 
2022  // Loop for both DGEMM and S/HGEMM 2nd instruction.
2023  for (const MachineOperand &Use : MI->explicit_uses()) {
2024  const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
2025  const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
2026  const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3;
2027  const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5;
2028  const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4;
2029  const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9;
2030  const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8;
2031  const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17;
2032  const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16;
2033  const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
2034  const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
2035  const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
2036  const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
2037  const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
2038  const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
2039  const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
2040  const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
2041  const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
2042  const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
2043  const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4;
2044  const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6;
2045  const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10;
2046  const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18;
2047  const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5;
2048  const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7;
2049  const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11;
2050  const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19;
2051  const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
2052  const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
2053  const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
2054  const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
2055  const int MaxWaitStates = 19;
2056 
2057  if (!Use.isReg())
2058  continue;
2059  Register Reg = Use.getReg();
2060  bool FullReg;
2061  const MachineInstr *MI1;
2062 
2063  auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
2064  this](const MachineInstr &MI) {
2065  if (!SIInstrInfo::isMFMA(MI))
2066  return false;
2067  Register DstReg = MI.getOperand(0).getReg();
2068  FullReg = (DstReg == Reg);
2069  MI1 = &MI;
2070  return TRI.regsOverlap(DstReg, Reg);
2071  };
2072 
2073  WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
2074  getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
2075  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2076 
2077  int NumWaitStates =
2078  getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
2079  if (NumWaitStates == std::numeric_limits<int>::max())
2080  continue;
2081 
2082  int OpNo = MI->getOperandNo(&Use);
2083  unsigned Opc1 = MI1->getOpcode();
2084  int NeedWaitStates = 0;
2085  if (OpNo == SrcCIdx) {
2086  if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
2087  NeedWaitStates = 0;
2088  } else if (FullReg) {
2089  if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2090  Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
2091  (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2092  Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
2093  NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
2094  else if (ST.hasGFX940Insts() &&
2095  TSchedModel.computeInstrLatency(MI1) == 2)
2096  NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
2097  } else {
2098  switch (Opc1) {
2099  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2100  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2101  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2102  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2103  if (!isXDL(ST, *MI))
2104  NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
2105  break;
2106  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2107  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2108  if (!isXDL(ST, *MI))
2109  NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
2110  break;
2111  default:
2112  if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1))
2113  break;
2114  switch (TSchedModel.computeInstrLatency(MI1)) {
2115  case 2:
2116  NeedWaitStates = ST.hasGFX940Insts()
2117  ? isXDL(ST, *MI1)
2118  ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates
2119  : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates
2120  : isDGEMM(Opc)
2121  ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
2122  : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
2123  break;
2124  case 4:
2125  assert(ST.hasGFX940Insts());
2126  NeedWaitStates = isXDL(ST, *MI1)
2127  ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates
2128  : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates;
2129  break;
2130  case 8:
2131  NeedWaitStates = ST.hasGFX940Insts()
2132  ? isXDL(ST, *MI1)
2133  ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates
2134  : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates
2135  : isDGEMM(Opc)
2136  ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
2137  : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
2138  break;
2139  case 16: LLVM_FALLTHROUGH;
2140  default:
2141  NeedWaitStates = ST.hasGFX940Insts()
2142  ? isXDL(ST, *MI1)
2143  ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates
2144  : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates
2145  : isDGEMM(Opc)
2146  ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
2147  : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
2148  }
2149  }
2150  }
2151  } else {
2152  switch (Opc1) {
2153  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2154  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2155  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2156  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2157  NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
2158  break;
2159  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2160  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2161  NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
2162  break;
2163  default:
2164  switch (TSchedModel.computeInstrLatency(MI1)) {
2165  case 2:
2166  NeedWaitStates = ST.hasGFX940Insts()
2167  ? isXDL(ST, *MI1)
2168  ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates
2169  : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates
2170  : SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
2171  break;
2172  case 4:
2173  assert(ST.hasGFX940Insts());
2174  NeedWaitStates = isXDL(ST, *MI1)
2175  ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates
2176  : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates;
2177  break;
2178  case 8:
2179  NeedWaitStates = ST.hasGFX940Insts()
2180  ? isXDL(ST, *MI1)
2181  ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates
2182  : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates
2183  : SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
2184  break;
2185  case 16: LLVM_FALLTHROUGH;
2186  default:
2187  NeedWaitStates = ST.hasGFX940Insts()
2188  ? isXDL(ST, *MI1)
2189  ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates
2190  : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates
2191  : SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
2192  }
2193  }
2194  }
2195  if (WaitStatesNeeded >= NeedWaitStates)
2196  continue;
2197 
2198  WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
2199  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2200 
2201  if (WaitStatesNeeded == MaxWaitStates)
2202  break;
2203  }
2204 
2205  return WaitStatesNeeded;
2206 }
2207 
2208 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
2209  // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
2210  if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
2211  return 0;
2212 
2213  int WaitStatesNeeded = 0;
2214 
2215  auto IsAccVgprReadFn = [](const MachineInstr &MI) {
2216  return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
2217  };
2218 
2219  for (const MachineOperand &Op : MI->explicit_uses()) {
2220  if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
2221  continue;
2222 
2223  Register Reg = Op.getReg();
2224 
2225  const int AccVgprReadLdStWaitStates = 2;
2226  const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
2227  const int MaxWaitStates = 2;
2228 
2229  int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
2230  getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
2231  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2232 
2233  if (WaitStatesNeeded == MaxWaitStates)
2234  return WaitStatesNeeded; // Early exit.
2235 
2236  auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
2237  if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
2238  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2239  return false;
2240  auto IsVALUFn = [](const MachineInstr &MI) {
2242  };
2243  return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
2245  };
2246 
2247  WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
2248  getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
2249  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2250  }
2251 
2252  return WaitStatesNeeded;
2253 }
2254 
2255 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
2256  if (!ST.hasGFX90AInsts())
2257  return 0;
2258 
2259  auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
2260  return isDGEMM(MI.getOpcode());
2261  };
2262 
2263  // This is checked in checkMAIHazards90A()
2264  if (SIInstrInfo::isMFMA(*MI))
2265  return 0;
2266 
2267  int WaitStatesNeeded = 0;
2268 
2269  bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
2270  SIInstrInfo::isFLAT(*MI) ||
2271  SIInstrInfo::isDS(*MI) ||
2273  bool IsVALU = SIInstrInfo::isVALU(*MI);
2274 
2275  const MachineInstr *MFMA = nullptr;
2276  unsigned Reg;
2277  auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2278  if (!SIInstrInfo::isMFMA(MI) ||
2279  !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2280  return false;
2281  MFMA = &MI;
2282  return true;
2283  };
2284 
2285  const MachineInstr *DOT = nullptr;
2286  auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
2287  if (!SIInstrInfo::isDOT(MI) ||
2288  !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2289  return false;
2290  DOT = &MI;
2291  return true;
2292  };
2293 
2294  int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
2295  AMDGPU::OpName::src2);
2296 
2297  if (IsMemOrExport || IsVALU) {
2298  const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
2299  const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
2300  const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
2301  const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4;
2302  const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6;
2303  const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10;
2304  const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18;
2305  const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5;
2306  const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7;
2307  const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11;
2308  const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19;
2309  const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
2310  const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
2311  const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
2312  const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
2313  const int DotWriteSameDotReadSrcAB = 3;
2314  const int DotWriteDifferentVALURead = 3;
2315  const int MaxWaitStates = 19;
2316 
2317  for (const MachineOperand &Use : MI->explicit_uses()) {
2318  if (!Use.isReg())
2319  continue;
2320  Reg = Use.getReg();
2321 
2322  DOT = nullptr;
2323  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2324  MaxWaitStates);
2325  if (DOT) {
2326  int NeedWaitStates = 0;
2327  if (DOT->getOpcode() == MI->getOpcode()) {
2328  if (&Use - &MI->getOperand(0) != SrcCIdx)
2329  NeedWaitStates = DotWriteSameDotReadSrcAB;
2330  } else {
2331  NeedWaitStates = DotWriteDifferentVALURead;
2332  }
2333 
2334  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2335  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2336  }
2337 
2338  MFMA = nullptr;
2339  WaitStatesSinceDef =
2340  getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2341  if (!MFMA)
2342  continue;
2343 
2344  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2345  int NeedWaitStates = MaxWaitStates;
2346  switch (HazardDefLatency) {
2347  case 2:
2348  NeedWaitStates =
2349  ST.hasGFX940Insts()
2350  ? isXDL(ST, *MFMA)
2351  ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates
2352  : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates
2353  : SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
2354  break;
2355  case 4:
2356  assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2357  NeedWaitStates =
2358  isDGEMM(MFMA->getOpcode())
2359  ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
2360  : DMFMA4x4WriteVgprVALUReadWaitStates
2361  : isXDL(ST, *MFMA)
2362  ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates
2363  : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates;
2364  break;
2365  case 8:
2366  NeedWaitStates =
2367  ST.hasGFX940Insts()
2368  ? isXDL(ST, *MFMA)
2369  ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates
2370  : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates
2371  : SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
2372  break;
2373  case 16: LLVM_FALLTHROUGH;
2374  default:
2375  NeedWaitStates =
2376  isDGEMM(MFMA->getOpcode())
2377  ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
2378  : DMFMA16x16WriteVgprVALUReadWaitStates
2379  : ST.hasGFX940Insts()
2380  ? isXDL(ST, *MFMA)
2381  ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates
2382  : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates
2383  : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
2384  break;
2385  }
2386 
2387  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2388  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2389 
2390  if (WaitStatesNeeded == MaxWaitStates)
2391  break;
2392  }
2393  }
2394 
2395  unsigned Opc = MI->getOpcode();
2396  const int DMFMAToFMA64WaitStates = 2;
2397  if ((Opc == AMDGPU::V_FMA_F64_e64 ||
2398  Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
2399  Opc == AMDGPU::V_FMAC_F64_dpp) &&
2400  WaitStatesNeeded < DMFMAToFMA64WaitStates) {
2401  int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
2402  getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
2403  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2404  }
2405 
2406  if (!IsVALU && !IsMemOrExport)
2407  return WaitStatesNeeded;
2408 
2409  for (const MachineOperand &Def : MI->defs()) {
2410  const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
2411  const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
2412  const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
2413  const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4;
2414  const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6;
2415  const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10;
2416  const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18;
2417  const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5;
2418  const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7;
2419  const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11;
2420  const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19;
2421  const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
2422  const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
2423  const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
2424  const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
2425  const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
2426  const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
2427  const int DotWriteDifferentVALUWrite = 3;
2428  const int MaxWaitStates = 19;
2429  const int MaxWarWaitStates = 15;
2430 
2431  Reg = Def.getReg();
2432 
2433  DOT = nullptr;
2434  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2435  MaxWaitStates);
2436  if (DOT && DOT->getOpcode() != MI->getOpcode())
2437  WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
2438  WaitStatesSinceDef);
2439 
2440  MFMA = nullptr;
2441  WaitStatesSinceDef =
2442  getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2443  if (MFMA) {
2444  int NeedWaitStates = MaxWaitStates;
2445  switch (TSchedModel.computeInstrLatency(MFMA)) {
2446  case 2:
2447  NeedWaitStates = ST.hasGFX940Insts()
2448  ? isXDL(ST, *MFMA)
2449  ? GFX940_XDL2PassWriteVgprVALUWawWaitStates
2450  : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates
2451  : SMFMA4x4WriteVgprVALUWawWaitStates;
2452  break;
2453  case 4:
2454  assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2455  NeedWaitStates = isDGEMM(MFMA->getOpcode())
2456  ? DMFMA4x4WriteVgprVALUWriteWaitStates
2457  : isXDL(ST, *MFMA)
2458  ? GFX940_XDL4PassWriteVgprVALUWawWaitStates
2459  : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates;
2460  break;
2461  case 8:
2462  NeedWaitStates = ST.hasGFX940Insts()
2463  ? isXDL(ST, *MFMA)
2464  ? GFX940_XDL8PassWriteVgprVALUWawWaitStates
2465  : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates
2466  : SMFMA16x16WriteVgprVALUWawWaitStates;
2467  break;
2468  case 16: LLVM_FALLTHROUGH;
2469  default:
2470  NeedWaitStates = isDGEMM(MFMA->getOpcode())
2471  ? DMFMA16x16WriteVgprVALUWriteWaitStates
2472  : ST.hasGFX940Insts()
2473  ? isXDL(ST, *MFMA)
2474  ? GFX940_XDL16PassWriteVgprVALUWawWaitStates
2475  : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates
2476  : SMFMA32x32WriteVgprVALUWawWaitStates;
2477  break;
2478  }
2479 
2480  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2481  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2482 
2483  if (WaitStatesNeeded == MaxWaitStates)
2484  break;
2485  }
2486 
2487  auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2488  if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
2489  !MI.readsRegister(Reg, &TRI))
2490  return false;
2491 
2492  if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2493  return false;
2494 
2495  const MachineOperand *SrcC =
2496  TII.getNamedOperand(MI, AMDGPU::OpName::src2);
2497  assert(SrcC);
2498  if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
2499  return false;
2500 
2501  MFMA = &MI;
2502  return true;
2503  };
2504 
2505  MFMA = nullptr;
2506  int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
2507  MaxWarWaitStates);
2508  if (!MFMA)
2509  continue;
2510 
2511  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2512  int NeedWaitStates = MaxWaitStates;
2513  switch (HazardDefLatency) {
2514  case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
2515  break;
2516  case 4: assert(ST.hasGFX940Insts());
2517  NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
2518  break;
2519  case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
2520  break;
2521  case 16: LLVM_FALLTHROUGH;
2522  default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
2523  break;
2524  }
2525 
2526  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
2527  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2528  }
2529 
2530  return WaitStatesNeeded;
2531 }
2532 
2534  if (!SU->isInstr())
2535  return false;
2536 
2537  const MachineInstr *MAI = nullptr;
2538 
2539  auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
2540  MAI = nullptr;
2541  if (SIInstrInfo::isMFMA(MI))
2542  MAI = &MI;
2543  return MAI != nullptr;
2544  };
2545 
2546  MachineInstr *MI = SU->getInstr();
2547  if (IsMFMAFn(*MI)) {
2548  int W = getWaitStatesSince(IsMFMAFn, 16);
2549  if (MAI)
2550  return W < (int)TSchedModel.computeInstrLatency(MAI);
2551  }
2552 
2553  return false;
2554 }
i
i
Definition: README.txt:29
llvm::GCNSubtarget::hasReadM0LdsDmaHazard
bool hasReadM0LdsDmaHazard() const
Definition: GCNSubtarget.h:964
llvm::GCNSubtarget::hasVDecCoExecHazard
bool hasVDecCoExecHazard() const
Definition: GCNSubtarget.h:1012
llvm::TargetSchedModel::getWriteProcResBegin
ProcResIter getWriteProcResBegin(const MCSchedClassDesc *SC) const
Definition: TargetSchedule.h:133
ScheduleDAG.h
llvm::MachineInstr::uses
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:667
llvm::ScheduleHazardRecognizer::getMaxLookAhead
unsigned getMaxLookAhead() const
Definition: ScheduleHazardRecognizer.h:43
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::AMDGPU::getMAIIsDGEMM
bool getMAIIsDGEMM(unsigned Opc)
Returns true if MAI operation is a double precision GEMM.
Definition: AMDGPUBaseInfo.cpp:391
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:581
llvm::GCNHazardRecognizer::getHazardType
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
Definition: GCNHazardRecognizer.cpp:181
llvm::GCNSubtarget::hasDstSelForwardingHazard
bool hasDstSelForwardingHazard() const
Definition: GCNSubtarget.h:1006
llvm::AMDGPU::getIsaVersion
IsaVersion getIsaVersion(StringRef GPU)
Definition: TargetParser.cpp:193
SIMachineFunctionInfo.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
NoHazardFound
@ NoHazardFound
Definition: GCNHazardRecognizer.cpp:427
llvm::SIInstrFlags::VALU
@ VALU
Definition: SIDefines.h:30
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:344
llvm::GCNSubtarget::hasVALUPartialForwardingHazard
bool hasVALUPartialForwardingHazard() const
Definition: GCNSubtarget.h:1030
llvm::BitVector::none
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:181
llvm::ScheduleHazardRecognizer::MaxLookAhead
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
Definition: ScheduleHazardRecognizer.h:31
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:77
llvm::SIInstrInfo::isBufferSMRD
bool isBufferSMRD(const MachineInstr &MI) const
Definition: SIInstrInfo.cpp:7738
llvm::SIInstrInfo::getNumWaitStates
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
Definition: SIInstrInfo.cpp:1764
llvm::RISCVFenceField::W
@ W
Definition: RISCVBaseInfo.h:242
llvm::GCNSubtarget::hasVMEMReadSGPRVALUDefHazard
bool hasVMEMReadSGPRVALUDefHazard() const
A read of an SGPR by a VMEM instruction requires 5 wait states when the SGPR was written by a VALU In...
Definition: GCNSubtarget.h:442
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::GCNHazardRecognizer::GCNHazardRecognizer
GCNHazardRecognizer(const MachineFunction &MF)
Definition: GCNHazardRecognizer.cpp:53
llvm::SmallDenseMap
Definition: DenseMap.h:882
llvm::SIInstrInfo::isEXP
static bool isEXP(const MachineInstr &MI)
Definition: SIInstrInfo.h:553
isRWLane
static bool isRWLane(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:98
llvm::GCNSubtarget::getSetRegWaitStates
unsigned getSetRegWaitStates() const
Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
Definition: GCNSubtarget.h:451
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::MachineOperand::setImm
void setImm(int64_t immVal)
Definition: MachineOperand.h:664
llvm::AMDGPU::Hwreg::ID_MASK_
@ ID_MASK_
Definition: SIDefines.h:413
llvm::ScheduleHazardRecognizer::Hazard
@ Hazard
Definition: ScheduleHazardRecognizer.h:39
isXDL
static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:122
addRegUnits
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
Definition: GCNHazardRecognizer.cpp:571
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:136
isSGetReg
static bool isSGetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:83
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
TargetParser.h
llvm::AMDGPU::getNamedOperandIdx
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
llvm::SIInstrFlags::TRANS
@ TRANS
Definition: SIDefines.h:51
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::insert
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
MFMAPaddingRatio
static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
llvm::SISrcMods::NEG
@ NEG
Definition: SIDefines.h:219
llvm::GCNSubtarget::getRegisterInfo
const SIRegisterInfo * getRegisterInfo() const override
Definition: GCNSubtarget.h:224
hasHazard
static bool hasHazard(StateT State, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, DenseSet< const MachineBasicBlock * > &Visited)
Definition: GCNHazardRecognizer.cpp:435
llvm::InlineAsm::MIOp_FirstOperand
@ MIOp_FirstOperand
Definition: InlineAsm.h:225
llvm::GCNSubtarget::hasLdsDirect
bool hasLdsDirect() const
Definition: GCNSubtarget.h:1028
llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition: GCNSubtarget.h:212
llvm::TargetSchedModel::init
void init(const TargetSubtargetInfo *TSInfo)
Initialize the machine model for instruction scheduling.
Definition: TargetSchedule.cpp:62
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
llvm::SIInstrInfo::isWMMA
static bool isWMMA(const MachineInstr &MI)
Definition: SIInstrInfo.h:676
HazardFound
@ HazardFound
Definition: GCNHazardRecognizer.cpp:427
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:666
llvm::SIInstrInfo::isMIMG
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:500
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:418
GCNSubtarget.h
getHWReg
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
Definition: GCNHazardRecognizer.cpp:174
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:546
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:754
llvm::SIInstrFlags::DPP
@ DPP
Definition: SIDefines.h:50
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:501
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::SIInstrInfo::isMAI
static bool isMAI(const MachineInstr &MI)
Definition: SIInstrInfo.h:659
llvm::MCWriteProcResEntry::Cycles
uint16_t Cycles
Definition: MCSchedule.h:65
llvm::MCSchedClassDesc
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Definition: MCSchedule.h:109
shouldRunLdsBranchVmemWARHazardFixup
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST)
Definition: GCNHazardRecognizer.cpp:1294
llvm::GCNSubtarget::hasSMRDReadVALUDefHazard
bool hasSMRDReadVALUDefHazard() const
A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR was written by a VALU inst...
Definition: GCNSubtarget.h:436
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
isRFE
static bool isRFE(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:102
llvm::AMDGPU::getMAIIsGFX940XDL
bool getMAIIsGFX940XDL(unsigned Opc)
Definition: AMDGPUBaseInfo.cpp:396
false
Definition: StackSlotColoring.cpp:141
llvm::GCNSubtarget::hasGFX940Insts
bool hasGFX940Insts() const
Definition: GCNSubtarget.h:1061
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::GraphProgram::DOT
@ DOT
Definition: GraphWriter.h:51
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
getWaitStatesSince
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
Definition: GCNHazardRecognizer.cpp:477
llvm::M0
unsigned M0(unsigned Val)
Definition: VE.h:369
llvm::cl::parser
Definition: CommandLine.h:797
llvm::AMDGPU::decodeWaitcnt
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
Definition: AMDGPUBaseInfo.cpp:1045
llvm::SIInstrFlags::SMRD
@ SMRD
Definition: SIDefines.h:56
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:30
llvm::TargetSchedModel::getWriteProcResEnd
ProcResIter getWriteProcResEnd(const MCSchedClassDesc *SC) const
Definition: TargetSchedule.h:137
llvm::BitVector
Definition: BitVector.h:75
llvm::MCOperandInfo::RegClass
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:90
llvm::GCNSubtarget::hasTransForwardingHazard
bool hasTransForwardingHazard() const
Definition: GCNSubtarget.h:1002
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::GCNSubtarget::hasVcmpxExecWARHazard
bool hasVcmpxExecWARHazard() const
Definition: GCNSubtarget.h:992
llvm::GCNSubtarget::isWave64
bool isWave64() const
Definition: GCNSubtarget.h:1238
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::TargetSchedModel::resolveSchedClass
const MCSchedClassDesc * resolveSchedClass(const MachineInstr *MI) const
Return the MCSchedClassDesc for this instruction.
Definition: TargetSchedule.cpp:132
isSendMsgTraceDataOrGDS
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:137
isPermlane
static bool isPermlane(const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:163
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition: DenseSet.h:268
llvm::cl::opt
Definition: CommandLine.h:1392
llvm::AMDGPU::getRegBitWidth
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
Definition: AMDGPUBaseInfo.cpp:1946
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:240
llvm::cl::Option
Definition: CommandLine.h:242
llvm::SIInstrInfo::isDPP
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:627
llvm::ScheduleHazardRecognizer::HazardType
HazardType
Definition: ScheduleHazardRecognizer.h:37
AMDGPUMCTargetDesc.h
llvm::TargetRegisterInfo::regsOverlap
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Definition: TargetRegisterInfo.h:419
llvm::GCNHazardRecognizer::PreEmitNoopsCommon
unsigned PreEmitNoopsCommon(MachineInstr *)
Definition: GCNHazardRecognizer.cpp:310
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
llvm::MachineBasicBlock::instr_rend
reverse_instr_iterator instr_rend()
Definition: MachineBasicBlock.h:268
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:320
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
UseReg
static Register UseReg(const MachineOperand &MO)
Definition: HexagonCopyToCombine.cpp:252
llvm::GCNSubtarget::hasVMEMtoScalarWriteHazard
bool hasVMEMtoScalarWriteHazard() const
Definition: GCNSubtarget.h:976
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SIInstrInfo::isDS
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:490
GetNumWaitStatesFn
function_ref< unsigned int(const MachineInstr &)> GetNumWaitStatesFn
Definition: GCNHazardRecognizer.cpp:430
llvm::SIInstrInfo::isSMRD
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:480
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SUnit::getInstr
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
isLdsDma
static bool isLdsDma(const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:169
llvm::AMDGPU::SDWA::DWORD
@ DWORD
Definition: SIDefines.h:772
addRegsToSet
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
Definition: GCNHazardRecognizer.cpp:577
llvm::GCNHazardRecognizer::PreEmitNoops
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
Definition: GCNHazardRecognizer.cpp:301
llvm::SIInstrInfo::isMFMA
static bool isMFMA(const MachineInstr &MI)
Definition: SIInstrInfo.h:667
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
HazardExpired
@ HazardExpired
Definition: GCNHazardRecognizer.cpp:427
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::SIInstrInfo::isFPAtomic
static bool isFPAtomic(const MachineInstr &MI)
Definition: SIInstrInfo.h:770
llvm::GCNSubtarget::hasReadM0MovRelInterpHazard
bool hasReadM0MovRelInterpHazard() const
Definition: GCNSubtarget.h:955
llvm::MCInstrDesc::OpInfo
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:208
llvm::GCNSubtarget::hasRFEHazards
bool hasRFEHazards() const
Definition: GCNSubtarget.h:446
HazardFnResult
HazardFnResult
Definition: GCNHazardRecognizer.cpp:427
llvm::GCNSubtarget::hasGFX90AInsts
bool hasGFX90AInsts() const
Definition: GCNSubtarget.h:1024
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::AMDGPU::getMIMGInfo
const LLVM_READONLY MIMGInfo * getMIMGInfo(unsigned Opc)
llvm::GCNSubtarget::hasNoDataDepHazard
bool hasNoDataDepHazard() const
Definition: GCNSubtarget.h:762
llvm::GCNHazardRecognizer::EmitInstruction
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
Definition: GCNHazardRecognizer.cpp:71
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::MachineBasicBlock::predecessors
iterator_range< pred_iterator > predecessors()
Definition: MachineBasicBlock.h:359
IsExpiredFn
function_ref< bool(const MachineInstr &, int WaitStates)> IsExpiredFn
Definition: GCNHazardRecognizer.cpp:429
llvm::GCNHazardRecognizer::EmitNoop
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
Definition: GCNHazardRecognizer.cpp:377
breaksVMEMSoftClause
static bool breaksVMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:596
llvm::GCNSubtarget::hasLdsBranchVmemWARHazard
bool hasLdsBranchVmemWARHazard() const
Definition: GCNSubtarget.h:996
llvm::SIMachineFunctionInfo::getOccupancy
unsigned getOccupancy() const
Definition: SIMachineFunctionInfo.h:952
llvm::SIInstrInfo::isSALU
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:352
llvm::MachineBasicBlock::instr_end
instr_iterator instr_end()
Definition: MachineBasicBlock.h:264
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::SISrcMods::NEG_HI
@ NEG_HI
Definition: SIDefines.h:222
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
insertNoopsInBundle
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
Definition: GCNHazardRecognizer.cpp:257
cl
http eax xorl edx cl sete al setne dl sall cl
Definition: README.txt:25
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::SIInstrInfo::isSDWA
static bool isSDWA(const MachineInstr &MI)
Definition: SIInstrInfo.h:448
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
parse
static llvm::Error parse(DataExtractor &Data, uint64_t BaseAddr, LineEntryCallback const &Callback)
Definition: LineTable.cpp:54
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
llvm::GCNSubtarget::hasVALUTransUseHazard
bool hasVALUTransUseHazard() const
Definition: GCNSubtarget.h:1034
isSSetReg
static bool isSSetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:87
llvm::SIInstrInfo::isMUBUF
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:464
llvm::SmallSet::insert
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:182
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:288
llvm::SIInstrInfo::isDOT
static bool isDOT(const MachineInstr &MI)
Definition: SIInstrInfo.h:672
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:280
llvm::GCNHazardRecognizer::Reset
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
Definition: GCNHazardRecognizer.cpp:67
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
isDGEMM
static bool isDGEMM(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:118
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::ScheduleHazardRecognizer::NoopHazard
@ NoopHazard
Definition: ScheduleHazardRecognizer.h:40
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::GCNSubtarget::hasNSAtoVMEMBug
bool hasNSAtoVMEMBug() const
Definition: GCNSubtarget.h:1016
llvm::GCNHazardRecognizer::RecedeCycle
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:419
llvm::ScheduleHazardRecognizer::NoHazard
@ NoHazard
Definition: ScheduleHazardRecognizer.h:38
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::SIInstrInfo::isSegmentSpecificFLAT
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:522
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::MachineInstr::mayStore
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:1025
llvm::SIInstrInfo::isVMEM
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:368
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::SIInstrInfo::isMTBUF
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:472
llvm::MachineInstr::isBundle
bool isBundle() const
Definition: MachineInstr.h:1288
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
llvm::SIInstrInfo
Definition: SIInstrInfo.h:44
GCNHazardRecognizer.h
llvm::GCNHazardRecognizer::AdvanceCycle
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:381
llvm::MCRegUnitIterator
Definition: MCRegisterInfo.h:680
llvm::GCNSubtarget::isXNACKEnabled
bool isXNACKEnabled() const
Definition: GCNSubtarget.h:541
isDivFMas
static bool isDivFMas(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:79
llvm::SUnit::isInstr
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
Definition: ScheduleDAG.h:362
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::SIInstrInfo::isVALU
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:360
llvm::MCRegisterInfo::DiffListIterator::isValid
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
Definition: MCRegisterInfo.h:224
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:494
llvm::SIMachineFunctionInfo
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Definition: SIMachineFunctionInfo.h:349
llvm::GCNHazardRecognizer::ShouldPreferAnother
bool ShouldPreferAnother(SUnit *SU) override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
Definition: GCNHazardRecognizer.cpp:2533
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::GCNHazardRecognizer::IsHazardFn
function_ref< bool(const MachineInstr &)> IsHazardFn
Definition: GCNHazardRecognizer.h:34
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::GCNSubtarget::hasReadM0SendMsgHazard
bool hasReadM0SendMsgHazard() const
Definition: GCNSubtarget.h:959
llvm::BitVector::anyCommon
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:482
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::SUnit
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
llvm::GCNSubtarget::hasSMEMtoVectorWriteHazard
bool hasSMEMtoVectorWriteHazard() const
Definition: GCNSubtarget.h:980
llvm::AMDGPU::Hwreg::ID_TRAPSTS
@ ID_TRAPSTS
Definition: SIDefines.h:388
llvm::SIInstrInfo::isTRANS
static bool isTRANS(const MachineInstr &MI)
Definition: SIInstrInfo.h:635
llvm::SIInstrInfo::isFLAT
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:516
llvm::cl::desc
Definition: CommandLine.h:405
llvm::SIInstrInfo::isLDSDIR
static bool isLDSDIR(const MachineInstr &MI)
Definition: SIInstrInfo.h:688
llvm::SISrcMods::DST_OP_SEL
@ DST_OP_SEL
Definition: SIDefines.h:225
breaksSMEMSoftClause
static bool breaksSMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:592
llvm::GCNSubtarget::has12DWordStoreHazard
bool has12DWordStoreHazard() const
Definition: GCNSubtarget.h:946
MachineFunction.h
llvm::GCNSubtarget::hasVcmpxPermlaneHazard
bool hasVcmpxPermlaneHazard() const
Definition: GCNSubtarget.h:972
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
isSMovRel
static bool isSMovRel(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:106
llvm::SIInstrInfo::isVINTRP
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:651
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::GCNSubtarget::hasMAIInsts
bool hasMAIInsts() const
Definition: GCNSubtarget.h:720
llvm::MCRegister
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
llvm::GCNSubtarget::hasReadM0LdsDirectHazard
bool hasReadM0LdsDirectHazard() const
Definition: GCNSubtarget.h:968
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52