LLVM  9.0.0svn
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "AMDGPUSubtarget.h"
15 #include "SIDefines.h"
16 #include "SIInstrInfo.h"
17 #include "SIRegisterInfo.h"
19 #include "Utils/AMDGPUBaseInfo.h"
26 #include "llvm/MC/MCInstrDesc.h"
28 #include <algorithm>
29 #include <cassert>
30 #include <limits>
31 #include <set>
32 #include <vector>
33 
34 using namespace llvm;
35 
36 //===----------------------------------------------------------------------===//
37 // Hazard Recoginizer Implementation
38 //===----------------------------------------------------------------------===//
39 
41  IsHazardRecognizerMode(false),
42  CurrCycleInstr(nullptr),
43  MF(MF),
44  ST(MF.getSubtarget<GCNSubtarget>()),
45  TII(*ST.getInstrInfo()),
46  TRI(TII.getRegisterInfo()),
47  ClauseUses(TRI.getNumRegUnits()),
48  ClauseDefs(TRI.getNumRegUnits()) {
49  MaxLookAhead = 5;
50 }
51 
54 }
55 
57  CurrCycleInstr = MI;
58 }
59 
60 static bool isDivFMas(unsigned Opcode) {
61  return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
62 }
63 
64 static bool isSGetReg(unsigned Opcode) {
65  return Opcode == AMDGPU::S_GETREG_B32;
66 }
67 
68 static bool isSSetReg(unsigned Opcode) {
69  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
70 }
71 
72 static bool isRWLane(unsigned Opcode) {
73  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
74 }
75 
76 static bool isRFE(unsigned Opcode) {
77  return Opcode == AMDGPU::S_RFE_B64;
78 }
79 
80 static bool isSMovRel(unsigned Opcode) {
81  switch (Opcode) {
82  case AMDGPU::S_MOVRELS_B32:
83  case AMDGPU::S_MOVRELS_B64:
84  case AMDGPU::S_MOVRELD_B32:
85  case AMDGPU::S_MOVRELD_B64:
86  return true;
87  default:
88  return false;
89  }
90 }
91 
92 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
93  const MachineInstr &MI) {
94  if (TII.isAlwaysGDS(MI.getOpcode()))
95  return true;
96 
97  switch (MI.getOpcode()) {
98  case AMDGPU::S_SENDMSG:
99  case AMDGPU::S_SENDMSGHALT:
100  case AMDGPU::S_TTRACEDATA:
101  return true;
102  // These DS opcodes don't support GDS.
103  case AMDGPU::DS_NOP:
104  case AMDGPU::DS_PERMUTE_B32:
105  case AMDGPU::DS_BPERMUTE_B32:
106  return false;
107  default:
108  if (TII.isDS(MI.getOpcode())) {
109  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
110  AMDGPU::OpName::gds);
111  if (MI.getOperand(GDS).getImm())
112  return true;
113  }
114  return false;
115  }
116 }
117 
118 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
119  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
120  AMDGPU::OpName::simm16);
121  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
122 }
123 
126  MachineInstr *MI = SU->getInstr();
127  if (MI->isBundle())
128  return NoHazard;
129 
130  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
131  return NoopHazard;
132 
133  // FIXME: Should flat be considered vmem?
134  if ((SIInstrInfo::isVMEM(*MI) ||
135  SIInstrInfo::isFLAT(*MI))
136  && checkVMEMHazards(MI) > 0)
137  return NoopHazard;
138 
139  if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
140  return NoopHazard;
141 
142  if (ST.hasNoDataDepHazard())
143  return NoHazard;
144 
145  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
146  return NoopHazard;
147 
148  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
149  return NoopHazard;
150 
151  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
152  return NoopHazard;
153 
154  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
155  return NoopHazard;
156 
157  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
158  return NoopHazard;
159 
160  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
161  return NoopHazard;
162 
163  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
164  return NoopHazard;
165 
166  if (ST.hasReadM0MovRelInterpHazard() &&
167  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
168  checkReadM0Hazards(MI) > 0)
169  return NoopHazard;
170 
171  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
172  checkReadM0Hazards(MI) > 0)
173  return NoopHazard;
174 
175  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
176  return NoopHazard;
177 
178  if (checkAnyInstHazards(MI) > 0)
179  return NoopHazard;
180 
181  return NoHazard;
182 }
183 
184 static void insertNoopInBundle(MachineInstr *MI, const SIInstrInfo &TII) {
185  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
186  .addImm(0);
187 }
188 
189 void GCNHazardRecognizer::processBundle() {
190  MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
192  // Check bundled MachineInstr's for hazards.
193  for (; MI != E && MI->isInsideBundle(); ++MI) {
194  CurrCycleInstr = &*MI;
195  unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
196 
197  if (IsHazardRecognizerMode)
198  fixHazards(CurrCycleInstr);
199 
200  for (unsigned i = 0; i < WaitStates; ++i)
201  insertNoopInBundle(CurrCycleInstr, TII);
202 
203  // It’s unnecessary to track more than MaxLookAhead instructions. Since we
204  // include the bundled MI directly after, only add a maximum of
205  // (MaxLookAhead - 1) noops to EmittedInstrs.
206  for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
207  EmittedInstrs.push_front(nullptr);
208 
209  EmittedInstrs.push_front(CurrCycleInstr);
210  EmittedInstrs.resize(MaxLookAhead);
211  }
212  CurrCycleInstr = nullptr;
213 }
214 
216  IsHazardRecognizerMode = false;
217  return PreEmitNoopsCommon(SU->getInstr());
218 }
219 
221  IsHazardRecognizerMode = true;
222  CurrCycleInstr = MI;
223  unsigned W = PreEmitNoopsCommon(MI);
224  fixHazards(MI);
225  CurrCycleInstr = nullptr;
226  return W;
227 }
228 
230  if (MI->isBundle())
231  return 0;
232 
233  int WaitStates = std::max(0, checkAnyInstHazards(MI));
234 
235  if (SIInstrInfo::isSMRD(*MI))
236  return std::max(WaitStates, checkSMRDHazards(MI));
237 
238  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
239  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
240 
241  if (ST.hasNSAtoVMEMBug())
242  WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
243 
244  if (ST.hasNoDataDepHazard())
245  return WaitStates;
246 
247  if (SIInstrInfo::isVALU(*MI))
248  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
249 
250  if (SIInstrInfo::isDPP(*MI))
251  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
252 
253  if (isDivFMas(MI->getOpcode()))
254  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
255 
256  if (isRWLane(MI->getOpcode()))
257  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
258 
259  if (MI->isInlineAsm())
260  return std::max(WaitStates, checkInlineAsmHazards(MI));
261 
262  if (isSGetReg(MI->getOpcode()))
263  return std::max(WaitStates, checkGetRegHazards(MI));
264 
265  if (isSSetReg(MI->getOpcode()))
266  return std::max(WaitStates, checkSetRegHazards(MI));
267 
268  if (isRFE(MI->getOpcode()))
269  return std::max(WaitStates, checkRFEHazards(MI));
270 
271  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
272  isSMovRel(MI->getOpcode())))
273  return std::max(WaitStates, checkReadM0Hazards(MI));
274 
275  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
276  return std::max(WaitStates, checkReadM0Hazards(MI));
277 
278  return WaitStates;
279 }
280 
282  EmittedInstrs.push_front(nullptr);
283 }
284 
286  // When the scheduler detects a stall, it will call AdvanceCycle() without
287  // emitting any instructions.
288  if (!CurrCycleInstr)
289  return;
290 
291  // Do not track non-instructions which do not affect the wait states.
292  // If included, these instructions can lead to buffer overflow such that
293  // detectable hazards are missed.
294  if (CurrCycleInstr->isImplicitDef() || CurrCycleInstr->isDebugInstr() ||
295  CurrCycleInstr->isKill())
296  return;
297 
298  if (CurrCycleInstr->isBundle()) {
299  processBundle();
300  return;
301  }
302 
303  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
304 
305  // Keep track of emitted instructions
306  EmittedInstrs.push_front(CurrCycleInstr);
307 
308  // Add a nullptr for each additional wait state after the first. Make sure
309  // not to add more than getMaxLookAhead() items to the list, since we
310  // truncate the list to that size right after this loop.
311  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
312  i < e; ++i) {
313  EmittedInstrs.push_front(nullptr);
314  }
315 
316  // getMaxLookahead() is the largest number of wait states we will ever need
317  // to insert, so there is no point in keeping track of more than that many
318  // wait states.
319  EmittedInstrs.resize(getMaxLookAhead());
320 
321  CurrCycleInstr = nullptr;
322 }
323 
325  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
326 }
327 
328 //===----------------------------------------------------------------------===//
329 // Helper Functions
330 //===----------------------------------------------------------------------===//
331 
333 
334 // Returns a minimum wait states since \p I walking all predecessors.
335 // Only scans until \p IsExpired does not return true.
336 // Can only be run in a hazard recognizer mode.
337 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
338  MachineBasicBlock *MBB,
340  int WaitStates,
341  IsExpiredFn IsExpired,
343  for (auto E = MBB->instr_rend(); I != E; ++I) {
344  // Don't add WaitStates for parent BUNDLE instructions.
345  if (I->isBundle())
346  continue;
347 
348  if (IsHazard(&*I))
349  return WaitStates;
350 
351  if (I->isInlineAsm() || I->isImplicitDef() || I->isDebugInstr())
352  continue;
353 
354  WaitStates += SIInstrInfo::getNumWaitStates(*I);
355 
356  if (IsExpired(&*I, WaitStates))
358  }
359 
360  int MinWaitStates = WaitStates;
361  bool Found = false;
362  for (MachineBasicBlock *Pred : MBB->predecessors()) {
363  if (!Visited.insert(Pred).second)
364  continue;
365 
366  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
367  WaitStates, IsExpired, Visited);
368 
370  continue;
371 
372  MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
373  if (IsExpired(nullptr, MinWaitStates))
374  return MinWaitStates;
375 
376  Found = true;
377  }
378 
379  if (Found)
380  return MinWaitStates;
381 
383 }
384 
385 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
386  MachineInstr *MI,
387  IsExpiredFn IsExpired) {
389  return getWaitStatesSince(IsHazard, MI->getParent(),
390  std::next(MI->getReverseIterator()),
391  0, IsExpired, Visited);
392 }
393 
394 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
395  if (IsHazardRecognizerMode) {
396  auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
397  return WaitStates >= Limit;
398  };
399  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
400  }
401 
402  int WaitStates = 0;
403  for (MachineInstr *MI : EmittedInstrs) {
404  if (MI) {
405  if (IsHazard(MI))
406  return WaitStates;
407 
408  if (MI->isInlineAsm())
409  continue;
410  }
411  ++WaitStates;
412 
413  if (WaitStates >= Limit)
414  break;
415  }
417 }
418 
419 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
420  IsHazardFn IsHazardDef,
421  int Limit) {
422  const SIRegisterInfo *TRI = ST.getRegisterInfo();
423 
424  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
425  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
426  };
427 
428  return getWaitStatesSince(IsHazardFn, Limit);
429 }
430 
431 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
432  int Limit) {
433  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
434  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
435  };
436 
437  return getWaitStatesSince(IsHazardFn, Limit);
438 }
439 
440 //===----------------------------------------------------------------------===//
441 // No-op Hazard Detection
442 //===----------------------------------------------------------------------===//
443 
444 static void addRegUnits(const SIRegisterInfo &TRI,
445  BitVector &BV, unsigned Reg) {
446  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
447  BV.set(*RUI);
448 }
449 
450 static void addRegsToSet(const SIRegisterInfo &TRI,
452  BitVector &Set) {
453  for (const MachineOperand &Op : Ops) {
454  if (Op.isReg())
455  addRegUnits(TRI, Set, Op.getReg());
456  }
457 }
458 
459 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
460  // XXX: Do we need to worry about implicit operands
461  addRegsToSet(TRI, MI.defs(), ClauseDefs);
462  addRegsToSet(TRI, MI.uses(), ClauseUses);
463 }
464 
465 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
466  // SMEM soft clause are only present on VI+, and only matter if xnack is
467  // enabled.
468  if (!ST.isXNACKEnabled())
469  return 0;
470 
471  bool IsSMRD = TII.isSMRD(*MEM);
472 
473  resetClause();
474 
475  // A soft-clause is any group of consecutive SMEM instructions. The
476  // instructions in this group may return out of order and/or may be
477  // replayed (i.e. the same instruction issued more than once).
478  //
479  // In order to handle these situations correctly we need to make sure that
480  // when a clause has more than one instruction, no instruction in the clause
481  // writes to a register that is read by another instruction in the clause
482  // (including itself). If we encounter this situaion, we need to break the
483  // clause by inserting a non SMEM instruction.
484 
485  for (MachineInstr *MI : EmittedInstrs) {
486  // When we hit a non-SMEM instruction then we have passed the start of the
487  // clause and we can stop.
488  if (!MI)
489  break;
490 
491  if (IsSMRD != SIInstrInfo::isSMRD(*MI))
492  break;
493 
494  addClauseInst(*MI);
495  }
496 
497  if (ClauseDefs.none())
498  return 0;
499 
500  // We need to make sure not to put loads and stores in the same clause if they
501  // use the same address. For now, just start a new clause whenever we see a
502  // store.
503  if (MEM->mayStore())
504  return 1;
505 
506  addClauseInst(*MEM);
507 
508  // If the set of defs and uses intersect then we cannot add this instruction
509  // to the clause, so we have a hazard.
510  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
511 }
512 
513 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
514  int WaitStatesNeeded = 0;
515 
516  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
517 
518  // This SMRD hazard only affects SI.
520  return WaitStatesNeeded;
521 
522  // A read of an SGPR by SMRD instruction requires 4 wait states when the
523  // SGPR was written by a VALU instruction.
524  int SmrdSgprWaitStates = 4;
525  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
526  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
527 
528  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
529 
530  for (const MachineOperand &Use : SMRD->uses()) {
531  if (!Use.isReg())
532  continue;
533  int WaitStatesNeededForUse =
534  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
535  SmrdSgprWaitStates);
536  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
537 
538  // This fixes what appears to be undocumented hardware behavior in SI where
539  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
540  // needs some number of nops in between. We don't know how many we need, but
541  // let's use 4. This wasn't discovered before probably because the only
542  // case when this happens is when we expand a 64-bit pointer into a full
543  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
544  // probably never encountered in the closed-source land.
545  if (IsBufferSMRD) {
546  int WaitStatesNeededForUse =
547  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
548  IsBufferHazardDefFn,
549  SmrdSgprWaitStates);
550  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
551  }
552  }
553 
554  return WaitStatesNeeded;
555 }
556 
557 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
559  return 0;
560 
561  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
562 
563  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
564  // SGPR was written by a VALU Instruction.
565  const int VmemSgprWaitStates = 5;
566  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
567  for (const MachineOperand &Use : VMEM->uses()) {
568  if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
569  continue;
570 
571  int WaitStatesNeededForUse =
572  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
573  VmemSgprWaitStates);
574  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
575  }
576  return WaitStatesNeeded;
577 }
578 
579 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
580  const SIRegisterInfo *TRI = ST.getRegisterInfo();
581  const SIInstrInfo *TII = ST.getInstrInfo();
582 
583  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
584  int DppVgprWaitStates = 2;
585  int DppExecWaitStates = 5;
586  int WaitStatesNeeded = 0;
587  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
588 
589  for (const MachineOperand &Use : DPP->uses()) {
590  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
591  continue;
592  int WaitStatesNeededForUse =
593  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
594  [](MachineInstr *) { return true; },
595  DppVgprWaitStates);
596  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
597  }
598 
599  WaitStatesNeeded = std::max(
600  WaitStatesNeeded,
601  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
602  DppExecWaitStates));
603 
604  return WaitStatesNeeded;
605 }
606 
607 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
608  const SIInstrInfo *TII = ST.getInstrInfo();
609 
610  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
611  // instruction.
612  const int DivFMasWaitStates = 4;
613  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
614  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
615  DivFMasWaitStates);
616 
617  return DivFMasWaitStates - WaitStatesNeeded;
618 }
619 
620 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
621  const SIInstrInfo *TII = ST.getInstrInfo();
622  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
623 
624  const int GetRegWaitStates = 2;
625  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
626  return GetRegHWReg == getHWReg(TII, *MI);
627  };
628  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
629 
630  return GetRegWaitStates - WaitStatesNeeded;
631 }
632 
633 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
634  const SIInstrInfo *TII = ST.getInstrInfo();
635  unsigned HWReg = getHWReg(TII, *SetRegInstr);
636 
637  const int SetRegWaitStates =
639  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
640  return HWReg == getHWReg(TII, *MI);
641  };
642  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
643  return SetRegWaitStates - WaitStatesNeeded;
644 }
645 
646 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
647  if (!MI.mayStore())
648  return -1;
649 
650  const SIInstrInfo *TII = ST.getInstrInfo();
651  unsigned Opcode = MI.getOpcode();
652  const MCInstrDesc &Desc = MI.getDesc();
653 
654  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
655  int VDataRCID = -1;
656  if (VDataIdx != -1)
657  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
658 
659  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
660  // There is no hazard if the instruction does not use vector regs
661  // (like wbinvl1)
662  if (VDataIdx == -1)
663  return -1;
664  // For MUBUF/MTBUF instructions this hazard only exists if the
665  // instruction is not using a register in the soffset field.
666  const MachineOperand *SOffset =
667  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
668  // If we have no soffset operand, then assume this field has been
669  // hardcoded to zero.
670  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
671  (!SOffset || !SOffset->isReg()))
672  return VDataIdx;
673  }
674 
675  // MIMG instructions create a hazard if they don't use a 256-bit T# and
676  // the store size is greater than 8 bytes and they have more than two bits
677  // of their dmask set.
678  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
679  if (TII->isMIMG(MI)) {
680  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
681  assert(SRsrcIdx != -1 &&
682  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
683  (void)SRsrcIdx;
684  }
685 
686  if (TII->isFLAT(MI)) {
687  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
688  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
689  return DataIdx;
690  }
691 
692  return -1;
693 }
694 
695 int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
696  const MachineRegisterInfo &MRI) {
697  // Helper to check for the hazard where VMEM instructions that store more than
698  // 8 bytes can have there store data over written by the next instruction.
699  const SIRegisterInfo *TRI = ST.getRegisterInfo();
700 
701  const int VALUWaitStates = 1;
702  int WaitStatesNeeded = 0;
703 
704  if (!TRI->isVGPR(MRI, Def.getReg()))
705  return WaitStatesNeeded;
706  unsigned Reg = Def.getReg();
707  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
708  int DataIdx = createsVALUHazard(*MI);
709  return DataIdx >= 0 &&
710  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
711  };
712  int WaitStatesNeededForDef =
713  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
714  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
715 
716  return WaitStatesNeeded;
717 }
718 
719 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
720  // This checks for the hazard where VMEM instructions that store more than
721  // 8 bytes can have there store data over written by the next instruction.
722  if (!ST.has12DWordStoreHazard())
723  return 0;
724 
725  const MachineRegisterInfo &MRI = MF.getRegInfo();
726  int WaitStatesNeeded = 0;
727 
728  for (const MachineOperand &Def : VALU->defs()) {
729  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
730  }
731 
732  return WaitStatesNeeded;
733 }
734 
735 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
736  // This checks for hazards associated with inline asm statements.
737  // Since inline asms can contain just about anything, we use this
738  // to call/leverage other check*Hazard routines. Note that
739  // this function doesn't attempt to address all possible inline asm
740  // hazards (good luck), but is a collection of what has been
741  // problematic thus far.
742 
743  // see checkVALUHazards()
744  if (!ST.has12DWordStoreHazard())
745  return 0;
746 
747  const MachineRegisterInfo &MRI = MF.getRegInfo();
748  int WaitStatesNeeded = 0;
749 
750  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
751  I != E; ++I) {
752  const MachineOperand &Op = IA->getOperand(I);
753  if (Op.isReg() && Op.isDef()) {
754  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
755  }
756  }
757 
758  return WaitStatesNeeded;
759 }
760 
761 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
762  const SIInstrInfo *TII = ST.getInstrInfo();
763  const SIRegisterInfo *TRI = ST.getRegisterInfo();
764  const MachineRegisterInfo &MRI = MF.getRegInfo();
765 
766  const MachineOperand *LaneSelectOp =
767  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
768 
769  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
770  return 0;
771 
772  unsigned LaneSelectReg = LaneSelectOp->getReg();
773  auto IsHazardFn = [TII] (MachineInstr *MI) {
774  return TII->isVALU(*MI);
775  };
776 
777  const int RWLaneWaitStates = 4;
778  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
779  RWLaneWaitStates);
780  return RWLaneWaitStates - WaitStatesSince;
781 }
782 
783 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
785  return 0;
786 
787  const SIInstrInfo *TII = ST.getInstrInfo();
788 
789  const int RFEWaitStates = 1;
790 
791  auto IsHazardFn = [TII] (MachineInstr *MI) {
792  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
793  };
794  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
795  return RFEWaitStates - WaitStatesNeeded;
796 }
797 
798 int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
799  if (MI->isDebugInstr())
800  return 0;
801 
802  const SIRegisterInfo *TRI = ST.getRegisterInfo();
803  if (!ST.hasSMovFedHazard())
804  return 0;
805 
806  // Check for any instruction reading an SGPR after a write from
807  // s_mov_fed_b32.
808  int MovFedWaitStates = 1;
809  int WaitStatesNeeded = 0;
810 
811  for (const MachineOperand &Use : MI->uses()) {
812  if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
813  continue;
814  auto IsHazardFn = [] (MachineInstr *MI) {
815  return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
816  };
817  int WaitStatesNeededForUse =
818  MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn,
819  MovFedWaitStates);
820  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
821  }
822 
823  return WaitStatesNeeded;
824 }
825 
826 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
827  const SIInstrInfo *TII = ST.getInstrInfo();
828  const int SMovRelWaitStates = 1;
829  auto IsHazardFn = [TII] (MachineInstr *MI) {
830  return TII->isSALU(*MI);
831  };
832  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
833  SMovRelWaitStates);
834 }
835 
836 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
837  fixVMEMtoScalarWriteHazards(MI);
838  fixSMEMtoVectorWriteHazards(MI);
839  fixVcmpxExecWARHazard(MI);
840  fixLdsBranchVmemWARHazard(MI);
841 }
842 
843 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
844  if (!ST.hasVMEMtoScalarWriteHazard())
845  return false;
846 
847  if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
848  return false;
849 
850  if (MI->getNumDefs() == 0)
851  return false;
852 
853  const SIRegisterInfo *TRI = ST.getRegisterInfo();
854 
855  auto IsHazardFn = [TRI, MI] (MachineInstr *I) {
856  if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isDS(*I) &&
858  return false;
859 
860  for (const MachineOperand &Def : MI->defs()) {
861  MachineOperand *Op = I->findRegisterUseOperand(Def.getReg(), false, TRI);
862  if (!Op || (Op->isImplicit() && Op->getReg() == AMDGPU::EXEC))
863  continue;
864  return true;
865  }
866  return false;
867  };
868 
869  auto IsExpiredFn = [] (MachineInstr *MI, int) {
870  return MI && (SIInstrInfo::isVALU(*MI) ||
871  (MI->getOpcode() == AMDGPU::S_WAITCNT &&
872  !MI->getOperand(0).getImm()));
873  };
874 
875  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
877  return false;
878 
879  const SIInstrInfo *TII = ST.getInstrInfo();
880  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
881  return true;
882 }
883 
884 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
885  if (!ST.hasSMEMtoVectorWriteHazard())
886  return false;
887 
888  if (!SIInstrInfo::isVALU(*MI))
889  return false;
890 
891  unsigned SDSTName;
892  switch (MI->getOpcode()) {
893  case AMDGPU::V_READLANE_B32:
894  case AMDGPU::V_READFIRSTLANE_B32:
895  SDSTName = AMDGPU::OpName::vdst;
896  break;
897  default:
898  SDSTName = AMDGPU::OpName::sdst;
899  break;
900  }
901 
902  const SIInstrInfo *TII = ST.getInstrInfo();
903  const SIRegisterInfo *TRI = ST.getRegisterInfo();
904  const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
905  const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
906  if (!SDST) {
907  for (const auto &MO : MI->implicit_operands()) {
908  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
909  SDST = &MO;
910  break;
911  }
912  }
913  }
914 
915  if (!SDST)
916  return false;
917 
918  const unsigned SDSTReg = SDST->getReg();
919  auto IsHazardFn = [SDSTReg, TRI] (MachineInstr *I) {
920  return SIInstrInfo::isSMRD(*I) && I->readsRegister(SDSTReg, TRI);
921  };
922 
923  auto IsExpiredFn = [TII, IV] (MachineInstr *MI, int) {
924  if (MI) {
925  if (TII->isSALU(*MI)) {
926  switch (MI->getOpcode()) {
927  case AMDGPU::S_SETVSKIP:
928  case AMDGPU::S_VERSION:
929  case AMDGPU::S_WAITCNT_VSCNT:
930  case AMDGPU::S_WAITCNT_VMCNT:
931  case AMDGPU::S_WAITCNT_EXPCNT:
932  // These instructions cannot not mitigate the hazard.
933  return false;
934  case AMDGPU::S_WAITCNT_LGKMCNT:
935  // Reducing lgkmcnt count to 0 always mitigates the hazard.
936  return (MI->getOperand(1).getImm() == 0) &&
937  (MI->getOperand(0).getReg() == AMDGPU::SGPR_NULL);
938  case AMDGPU::S_WAITCNT: {
939  const int64_t Imm = MI->getOperand(0).getImm();
940  AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
941  return (Decoded.LgkmCnt == 0);
942  }
943  default:
944  // SOPP instructions cannot mitigate the hazard.
945  if (TII->isSOPP(*MI))
946  return false;
947  // At this point the SALU can be assumed to mitigate the hazard
948  // because either:
949  // (a) it is independent of the at risk SMEM (breaking chain),
950  // or
951  // (b) it is dependent on the SMEM, in which case an appropriate
952  // s_waitcnt lgkmcnt _must_ exist between it and the at risk
953  // SMEM instruction.
954  return true;
955  }
956  }
957  }
958  return false;
959  };
960 
961  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
963  return false;
964 
965  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
966  TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
967  .addImm(0);
968  return true;
969 }
970 
971 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
972  if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
973  return false;
974 
975  const SIRegisterInfo *TRI = ST.getRegisterInfo();
976  if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
977  return false;
978 
979  auto IsHazardFn = [TRI] (MachineInstr *I) {
980  if (SIInstrInfo::isVALU(*I))
981  return false;
982  return I->readsRegister(AMDGPU::EXEC, TRI);
983  };
984 
985  const SIInstrInfo *TII = ST.getInstrInfo();
986  auto IsExpiredFn = [TII, TRI] (MachineInstr *MI, int) {
987  if (!MI)
988  return false;
989  if (SIInstrInfo::isVALU(*MI)) {
990  if (TII->getNamedOperand(*MI, AMDGPU::OpName::sdst))
991  return true;
992  for (auto MO : MI->implicit_operands())
993  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
994  return true;
995  }
996  if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
997  (MI->getOperand(0).getImm() & 0xfffe) == 0xfffe)
998  return true;
999  return false;
1000  };
1001 
1002  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1004  return false;
1005 
1006  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1007  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1008  .addImm(0xfffe);
1009  return true;
1010 }
1011 
1012 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1013  if (!ST.hasLdsBranchVmemWARHazard())
1014  return false;
1015 
1016  auto IsHazardInst = [] (const MachineInstr *MI) {
1017  if (SIInstrInfo::isDS(*MI))
1018  return 1;
1020  return 2;
1021  return 0;
1022  };
1023 
1024  auto InstType = IsHazardInst(MI);
1025  if (!InstType)
1026  return false;
1027 
1028  auto IsExpiredFn = [&IsHazardInst] (MachineInstr *I, int) {
1029  return I && (IsHazardInst(I) ||
1030  (I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1031  I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1032  !I->getOperand(1).getImm()));
1033  };
1034 
1035  auto IsHazardFn = [InstType, &IsHazardInst] (MachineInstr *I) {
1036  if (!I->isBranch())
1037  return false;
1038 
1039  auto IsHazardFn = [InstType, IsHazardInst] (MachineInstr *I) {
1040  auto InstType2 = IsHazardInst(I);
1041  return InstType2 && InstType != InstType2;
1042  };
1043 
1044  auto IsExpiredFn = [InstType, &IsHazardInst] (MachineInstr *I, int) {
1045  if (!I)
1046  return false;
1047 
1048  auto InstType2 = IsHazardInst(I);
1049  if (InstType == InstType2)
1050  return true;
1051 
1052  return I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1053  I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1054  !I->getOperand(1).getImm();
1055  };
1056 
1059  };
1060 
1061  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1063  return false;
1064 
1065  const SIInstrInfo *TII = ST.getInstrInfo();
1066  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1067  TII->get(AMDGPU::S_WAITCNT_VSCNT))
1068  .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1069  .addImm(0);
1070 
1071  return true;
1072 }
1073 
1074 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1075  int NSAtoVMEMWaitStates = 1;
1076 
1077  if (!ST.hasNSAtoVMEMBug())
1078  return 0;
1079 
1080  if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1081  return 0;
1082 
1083  const SIInstrInfo *TII = ST.getInstrInfo();
1084  const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1085  if (!Offset || (Offset->getImm() & 6) == 0)
1086  return 0;
1087 
1088  auto IsHazardFn = [TII] (MachineInstr *I) {
1089  if (!SIInstrInfo::isMIMG(*I))
1090  return false;
1091  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I->getOpcode());
1092  return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1093  TII->getInstSizeInBytes(*I) >= 16;
1094  };
1095 
1096  return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1097 }
static bool isSMovRel(unsigned Opcode)
BitVector & set()
Definition: BitVector.h:397
Interface definition for SIRegisterInfo.
bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
AMDGPU specific subclass of TargetSubtarget.
instr_iterator instr_end()
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
This class represents lattice values for constants.
Definition: AllocatorList.h:23
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:491
bool hasReadM0MovRelInterpHazard() const
Implements a dense probed hash-table based set.
Definition: DenseSet.h:249
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:382
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:554
bool hasNSAtoVMEMBug() const
unsigned Reg
bool isInlineAsm() const
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:116
static bool isRFE(unsigned Opcode)
const SIInstrInfo * getInstrInfo() const override
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, unsigned Reg)
unsigned const TargetRegisterInfo * TRI
static bool isSOPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:374
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:446
function_ref< bool(MachineInstr *)> IsHazardFn
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:456
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:482
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
const HexagonInstrInfo * TII
bool isAlwaysGDS(uint16_t Opcode) const
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:411
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:466
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
bool hasLdsBranchVmemWARHazard() const
bool hasSMovFedHazard() const
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:326
bool isSGPRClass(const TargetRegisterClass *RC) const
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:405
bool isBundle() const
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:430
static bool isSSetReg(unsigned Opcode)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
unsigned PreEmitNoops(SUnit *SU) override
PreEmitNoops - This callback is invoked prior to emitting an instruction.
bool hasVMEMtoScalarWriteHazard() const
function_ref< bool(MachineInstr *, int WaitStates)> IsExpiredFn
bool hasNoDataDepHazard() const
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
Definition: MachineInstr.h:656
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:538
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:819
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
unsigned const MachineRegisterInfo * MRI
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:523
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void insertNoopInBundle(MachineInstr *MI, const SIInstrInfo &TII)
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, MachineBasicBlock *MBB, MachineBasicBlock::reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock *> &Visited)
bool hasSMEMtoVectorWriteHazard() const
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:480
Generation getGeneration() const
self_iterator getIterator()
Definition: ilist_node.h:81
iterator_range< pred_iterator > predecessors()
bool isImplicitDef() const
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
bool isDebugInstr() const
Definition: MachineInstr.h:998
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
IsaVersion getIsaVersion(StringRef GPU)
bool hasVcmpxExecWARHazard() const
Iterator for intrusive lists based on ilist_node.
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:318
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
MachineOperand class - Representation of each machine instruction operand.
GCNHazardRecognizer(const MachineFunction &MF)
int64_t getImm() const
reverse_instr_iterator instr_rend()
A range adaptor for a pair of iterators.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
iterator_range< mop_iterator > implicit_operands()
Definition: MachineInstr.h:472
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:488
static bool isRWLane(unsigned Opcode)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:201
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
Interface definition for SIInstrInfo.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
reverse_self_iterator getReverseIterator()
Definition: ilist_node.h:84
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:438
bool has12DWordStoreHazard() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:72
bool isXNACKEnabled() const
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:334
unsigned getNumDefs() const
Returns the total number of definitions.
Definition: MachineInstr.h:423
bool isKill() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSGetReg(unsigned Opcode)
const TargetRegisterClass * getPhysRegClass(unsigned Reg) const
Return the &#39;base&#39; register class for this register.
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
IRTranslator LLVM IR MI
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
bool hasReadM0SendMsgHazard() const
bool isBufferSMRD(const MachineInstr &MI) const
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
bool isImplicit() const
unsigned PreEmitNoopsCommon(MachineInstr *)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
const SIRegisterInfo * getRegisterInfo() const override
static bool isDivFMas(unsigned Opcode)