LLVM  9.0.0svn
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "AMDGPUSubtarget.h"
15 #include "SIDefines.h"
16 #include "SIInstrInfo.h"
17 #include "SIRegisterInfo.h"
19 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/MC/MCInstrDesc.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <limits>
30 #include <set>
31 #include <vector>
32 
33 using namespace llvm;
34 
35 //===----------------------------------------------------------------------===//
36 // Hazard Recoginizer Implementation
37 //===----------------------------------------------------------------------===//
38 
40  IsHazardRecognizerMode(false),
41  CurrCycleInstr(nullptr),
42  MF(MF),
43  ST(MF.getSubtarget<GCNSubtarget>()),
44  TII(*ST.getInstrInfo()),
45  TRI(TII.getRegisterInfo()),
46  ClauseUses(TRI.getNumRegUnits()),
47  ClauseDefs(TRI.getNumRegUnits()) {
48  MaxLookAhead = 5;
49 }
50 
53 }
54 
56  CurrCycleInstr = MI;
57 }
58 
59 static bool isDivFMas(unsigned Opcode) {
60  return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
61 }
62 
63 static bool isSGetReg(unsigned Opcode) {
64  return Opcode == AMDGPU::S_GETREG_B32;
65 }
66 
67 static bool isSSetReg(unsigned Opcode) {
68  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
69 }
70 
71 static bool isRWLane(unsigned Opcode) {
72  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
73 }
74 
75 static bool isRFE(unsigned Opcode) {
76  return Opcode == AMDGPU::S_RFE_B64;
77 }
78 
79 static bool isSMovRel(unsigned Opcode) {
80  switch (Opcode) {
81  case AMDGPU::S_MOVRELS_B32:
82  case AMDGPU::S_MOVRELS_B64:
83  case AMDGPU::S_MOVRELD_B32:
84  case AMDGPU::S_MOVRELD_B64:
85  return true;
86  default:
87  return false;
88  }
89 }
90 
91 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
92  const MachineInstr &MI) {
93  if (TII.isAlwaysGDS(MI.getOpcode()))
94  return true;
95 
96  switch (MI.getOpcode()) {
97  case AMDGPU::S_SENDMSG:
98  case AMDGPU::S_SENDMSGHALT:
99  case AMDGPU::S_TTRACEDATA:
100  return true;
101  // These DS opcodes don't support GDS.
102  case AMDGPU::DS_NOP:
103  case AMDGPU::DS_PERMUTE_B32:
104  case AMDGPU::DS_BPERMUTE_B32:
105  return false;
106  default:
107  if (TII.isDS(MI.getOpcode())) {
108  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
109  AMDGPU::OpName::gds);
110  if (MI.getOperand(GDS).getImm())
111  return true;
112  }
113  return false;
114  }
115 }
116 
117 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
118  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
119  AMDGPU::OpName::simm16);
120  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
121 }
122 
125  MachineInstr *MI = SU->getInstr();
126 
127  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
128  return NoopHazard;
129 
130  // FIXME: Should flat be considered vmem?
131  if ((SIInstrInfo::isVMEM(*MI) ||
132  SIInstrInfo::isFLAT(*MI))
133  && checkVMEMHazards(MI) > 0)
134  return NoopHazard;
135 
136  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
137  return NoopHazard;
138 
139  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
140  return NoopHazard;
141 
142  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
143  return NoopHazard;
144 
145  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
146  return NoopHazard;
147 
148  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
149  return NoopHazard;
150 
151  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
152  return NoopHazard;
153 
154  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
155  return NoopHazard;
156 
157  if (ST.hasReadM0MovRelInterpHazard() &&
158  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
159  checkReadM0Hazards(MI) > 0)
160  return NoopHazard;
161 
162  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
163  checkReadM0Hazards(MI) > 0)
164  return NoopHazard;
165 
166  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
167  return NoopHazard;
168 
169  if (checkAnyInstHazards(MI) > 0)
170  return NoopHazard;
171 
172  return NoHazard;
173 }
174 
176  IsHazardRecognizerMode = false;
177  return PreEmitNoopsCommon(SU->getInstr());
178 }
179 
181  IsHazardRecognizerMode = true;
182  CurrCycleInstr = MI;
183  unsigned W = PreEmitNoopsCommon(MI);
184  CurrCycleInstr = nullptr;
185  return W;
186 }
187 
189  int WaitStates = std::max(0, checkAnyInstHazards(MI));
190 
191  if (SIInstrInfo::isSMRD(*MI))
192  return std::max(WaitStates, checkSMRDHazards(MI));
193 
194  if (SIInstrInfo::isVALU(*MI))
195  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
196 
197  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
198  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
199 
200  if (SIInstrInfo::isDPP(*MI))
201  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
202 
203  if (isDivFMas(MI->getOpcode()))
204  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
205 
206  if (isRWLane(MI->getOpcode()))
207  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
208 
209  if (MI->isInlineAsm())
210  return std::max(WaitStates, checkInlineAsmHazards(MI));
211 
212  if (isSGetReg(MI->getOpcode()))
213  return std::max(WaitStates, checkGetRegHazards(MI));
214 
215  if (isSSetReg(MI->getOpcode()))
216  return std::max(WaitStates, checkSetRegHazards(MI));
217 
218  if (isRFE(MI->getOpcode()))
219  return std::max(WaitStates, checkRFEHazards(MI));
220 
221  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
222  isSMovRel(MI->getOpcode())))
223  return std::max(WaitStates, checkReadM0Hazards(MI));
224 
225  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
226  return std::max(WaitStates, checkReadM0Hazards(MI));
227 
228  return WaitStates;
229 }
230 
232  EmittedInstrs.push_front(nullptr);
233 }
234 
236  // When the scheduler detects a stall, it will call AdvanceCycle() without
237  // emitting any instructions.
238  if (!CurrCycleInstr)
239  return;
240 
241  // Do not track non-instructions which do not affect the wait states.
242  // If included, these instructions can lead to buffer overflow such that
243  // detectable hazards are missed.
244  if (CurrCycleInstr->isImplicitDef())
245  return;
246  else if (CurrCycleInstr->isDebugInstr())
247  return;
248 
249  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
250 
251  // Keep track of emitted instructions
252  EmittedInstrs.push_front(CurrCycleInstr);
253 
254  // Add a nullptr for each additional wait state after the first. Make sure
255  // not to add more than getMaxLookAhead() items to the list, since we
256  // truncate the list to that size right after this loop.
257  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
258  i < e; ++i) {
259  EmittedInstrs.push_front(nullptr);
260  }
261 
262  // getMaxLookahead() is the largest number of wait states we will ever need
263  // to insert, so there is no point in keeping track of more than that many
264  // wait states.
265  EmittedInstrs.resize(getMaxLookAhead());
266 
267  CurrCycleInstr = nullptr;
268 }
269 
271  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
272 }
273 
274 //===----------------------------------------------------------------------===//
275 // Helper Functions
276 //===----------------------------------------------------------------------===//
277 
279 
280 // Returns a minimum wait states since \p I walking all predecessors.
281 // Only scans until \p IsExpired does not return true.
282 // Can only be run in a hazard recognizer mode.
283 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
284  MachineBasicBlock *MBB,
286  int WaitStates,
287  IsExpiredFn IsExpired,
289 
290  for (auto E = MBB->rend() ; I != E; ++I) {
291  if (IsHazard(&*I))
292  return WaitStates;
293 
294  if (I->isInlineAsm() || I->isImplicitDef() || I->isDebugInstr())
295  continue;
296 
297  WaitStates += SIInstrInfo::getNumWaitStates(*I);
298 
299  if (IsExpired(&*I, WaitStates))
301  }
302 
303  int MinWaitStates = WaitStates;
304  bool Found = false;
305  for (MachineBasicBlock *Pred : MBB->predecessors()) {
306  if (!Visited.insert(Pred).second)
307  continue;
308 
309  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
310  WaitStates, IsExpired, Visited);
311 
313  continue;
314 
315  MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
316  if (IsExpired(nullptr, MinWaitStates))
317  return MinWaitStates;
318 
319  Found = true;
320  }
321 
322  if (Found)
323  return MinWaitStates;
324 
326 }
327 
328 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
329  MachineInstr *MI,
330  IsExpiredFn IsExpired) {
332  return getWaitStatesSince(IsHazard, MI->getParent(),
333  std::next(MI->getReverseIterator()),
334  0, IsExpired, Visited);
335 }
336 
337 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
338  if (IsHazardRecognizerMode) {
339  auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
340  return WaitStates >= Limit;
341  };
342  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
343  }
344 
345  int WaitStates = 0;
346  for (MachineInstr *MI : EmittedInstrs) {
347  if (MI) {
348  if (IsHazard(MI))
349  return WaitStates;
350 
351  if (MI->isInlineAsm())
352  continue;
353  }
354  ++WaitStates;
355 
356  if (WaitStates >= Limit)
357  break;
358  }
360 }
361 
362 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
363  IsHazardFn IsHazardDef,
364  int Limit) {
365  const SIRegisterInfo *TRI = ST.getRegisterInfo();
366 
367  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
368  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
369  };
370 
371  return getWaitStatesSince(IsHazardFn, Limit);
372 }
373 
374 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
375  int Limit) {
376  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
377  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
378  };
379 
380  return getWaitStatesSince(IsHazardFn, Limit);
381 }
382 
383 //===----------------------------------------------------------------------===//
384 // No-op Hazard Detection
385 //===----------------------------------------------------------------------===//
386 
387 static void addRegUnits(const SIRegisterInfo &TRI,
388  BitVector &BV, unsigned Reg) {
389  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
390  BV.set(*RUI);
391 }
392 
393 static void addRegsToSet(const SIRegisterInfo &TRI,
395  BitVector &Set) {
396  for (const MachineOperand &Op : Ops) {
397  if (Op.isReg())
398  addRegUnits(TRI, Set, Op.getReg());
399  }
400 }
401 
402 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
403  // XXX: Do we need to worry about implicit operands
404  addRegsToSet(TRI, MI.defs(), ClauseDefs);
405  addRegsToSet(TRI, MI.uses(), ClauseUses);
406 }
407 
408 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
409  // SMEM soft clause are only present on VI+, and only matter if xnack is
410  // enabled.
411  if (!ST.isXNACKEnabled())
412  return 0;
413 
414  bool IsSMRD = TII.isSMRD(*MEM);
415 
416  resetClause();
417 
418  // A soft-clause is any group of consecutive SMEM instructions. The
419  // instructions in this group may return out of order and/or may be
420  // replayed (i.e. the same instruction issued more than once).
421  //
422  // In order to handle these situations correctly we need to make sure
423  // that when a clause has more than one instruction, no instruction in the
424  // clause writes to a register that is read another instruction in the clause
425  // (including itself). If we encounter this situaion, we need to break the
426  // clause by inserting a non SMEM instruction.
427 
428  for (MachineInstr *MI : EmittedInstrs) {
429  // When we hit a non-SMEM instruction then we have passed the start of the
430  // clause and we can stop.
431  if (!MI)
432  break;
433 
434  if (IsSMRD != SIInstrInfo::isSMRD(*MI))
435  break;
436 
437  addClauseInst(*MI);
438  }
439 
440  if (ClauseDefs.none())
441  return 0;
442 
443  // We need to make sure not to put loads and stores in the same clause if they
444  // use the same address. For now, just start a new clause whenever we see a
445  // store.
446  if (MEM->mayStore())
447  return 1;
448 
449  addClauseInst(*MEM);
450 
451  // If the set of defs and uses intersect then we cannot add this instruction
452  // to the clause, so we have a hazard.
453  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
454 }
455 
456 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
457  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
458  int WaitStatesNeeded = 0;
459 
460  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
461 
462  // This SMRD hazard only affects SI.
464  return WaitStatesNeeded;
465 
466  // A read of an SGPR by SMRD instruction requires 4 wait states when the
467  // SGPR was written by a VALU instruction.
468  int SmrdSgprWaitStates = 4;
469  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
470  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
471 
472  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
473 
474  for (const MachineOperand &Use : SMRD->uses()) {
475  if (!Use.isReg())
476  continue;
477  int WaitStatesNeededForUse =
478  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
479  SmrdSgprWaitStates);
480  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
481 
482  // This fixes what appears to be undocumented hardware behavior in SI where
483  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
484  // needs some number of nops in between. We don't know how many we need, but
485  // let's use 4. This wasn't discovered before probably because the only
486  // case when this happens is when we expand a 64-bit pointer into a full
487  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
488  // probably never encountered in the closed-source land.
489  if (IsBufferSMRD) {
490  int WaitStatesNeededForUse =
491  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
492  IsBufferHazardDefFn,
493  SmrdSgprWaitStates);
494  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
495  }
496  }
497 
498  return WaitStatesNeeded;
499 }
500 
501 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
503  return 0;
504 
505  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
506 
507  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
508  // SGPR was written by a VALU Instruction.
509  const int VmemSgprWaitStates = 5;
510  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
511 
512  for (const MachineOperand &Use : VMEM->uses()) {
513  if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
514  continue;
515 
516  int WaitStatesNeededForUse =
517  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
518  VmemSgprWaitStates);
519  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
520  }
521  return WaitStatesNeeded;
522 }
523 
524 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
525  const SIRegisterInfo *TRI = ST.getRegisterInfo();
526  const SIInstrInfo *TII = ST.getInstrInfo();
527 
528  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
529  int DppVgprWaitStates = 2;
530  int DppExecWaitStates = 5;
531  int WaitStatesNeeded = 0;
532  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
533 
534  for (const MachineOperand &Use : DPP->uses()) {
535  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
536  continue;
537  int WaitStatesNeededForUse =
538  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
539  [](MachineInstr *) { return true; },
540  DppVgprWaitStates);
541  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
542  }
543 
544  WaitStatesNeeded = std::max(
545  WaitStatesNeeded,
546  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
547  DppExecWaitStates));
548 
549  return WaitStatesNeeded;
550 }
551 
552 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
553  const SIInstrInfo *TII = ST.getInstrInfo();
554 
555  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
556  // instruction.
557  const int DivFMasWaitStates = 4;
558  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
559  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
560  DivFMasWaitStates);
561 
562  return DivFMasWaitStates - WaitStatesNeeded;
563 }
564 
565 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
566  const SIInstrInfo *TII = ST.getInstrInfo();
567  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
568 
569  const int GetRegWaitStates = 2;
570  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
571  return GetRegHWReg == getHWReg(TII, *MI);
572  };
573  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
574 
575  return GetRegWaitStates - WaitStatesNeeded;
576 }
577 
578 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
579  const SIInstrInfo *TII = ST.getInstrInfo();
580  unsigned HWReg = getHWReg(TII, *SetRegInstr);
581 
582  const int SetRegWaitStates =
584  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
585  return HWReg == getHWReg(TII, *MI);
586  };
587  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
588  return SetRegWaitStates - WaitStatesNeeded;
589 }
590 
591 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
592  if (!MI.mayStore())
593  return -1;
594 
595  const SIInstrInfo *TII = ST.getInstrInfo();
596  unsigned Opcode = MI.getOpcode();
597  const MCInstrDesc &Desc = MI.getDesc();
598 
599  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
600  int VDataRCID = -1;
601  if (VDataIdx != -1)
602  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
603 
604  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
605  // There is no hazard if the instruction does not use vector regs
606  // (like wbinvl1)
607  if (VDataIdx == -1)
608  return -1;
609  // For MUBUF/MTBUF instructions this hazard only exists if the
610  // instruction is not using a register in the soffset field.
611  const MachineOperand *SOffset =
612  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
613  // If we have no soffset operand, then assume this field has been
614  // hardcoded to zero.
615  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
616  (!SOffset || !SOffset->isReg()))
617  return VDataIdx;
618  }
619 
620  // MIMG instructions create a hazard if they don't use a 256-bit T# and
621  // the store size is greater than 8 bytes and they have more than two bits
622  // of their dmask set.
623  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
624  if (TII->isMIMG(MI)) {
625  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
626  assert(SRsrcIdx != -1 &&
627  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
628  (void)SRsrcIdx;
629  }
630 
631  if (TII->isFLAT(MI)) {
632  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
633  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
634  return DataIdx;
635  }
636 
637  return -1;
638 }
639 
640 int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
641  const MachineRegisterInfo &MRI) {
642  // Helper to check for the hazard where VMEM instructions that store more than
643  // 8 bytes can have there store data over written by the next instruction.
644  const SIRegisterInfo *TRI = ST.getRegisterInfo();
645 
646  const int VALUWaitStates = 1;
647  int WaitStatesNeeded = 0;
648 
649  if (!TRI->isVGPR(MRI, Def.getReg()))
650  return WaitStatesNeeded;
651  unsigned Reg = Def.getReg();
652  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
653  int DataIdx = createsVALUHazard(*MI);
654  return DataIdx >= 0 &&
655  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
656  };
657  int WaitStatesNeededForDef =
658  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
659  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
660 
661  return WaitStatesNeeded;
662 }
663 
664 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
665  // This checks for the hazard where VMEM instructions that store more than
666  // 8 bytes can have there store data over written by the next instruction.
667  if (!ST.has12DWordStoreHazard())
668  return 0;
669 
670  const MachineRegisterInfo &MRI = MF.getRegInfo();
671  int WaitStatesNeeded = 0;
672 
673  for (const MachineOperand &Def : VALU->defs()) {
674  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
675  }
676 
677  return WaitStatesNeeded;
678 }
679 
680 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
681  // This checks for hazards associated with inline asm statements.
682  // Since inline asms can contain just about anything, we use this
683  // to call/leverage other check*Hazard routines. Note that
684  // this function doesn't attempt to address all possible inline asm
685  // hazards (good luck), but is a collection of what has been
686  // problematic thus far.
687 
688  // see checkVALUHazards()
689  if (!ST.has12DWordStoreHazard())
690  return 0;
691 
692  const MachineRegisterInfo &MRI = MF.getRegInfo();
693  int WaitStatesNeeded = 0;
694 
695  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
696  I != E; ++I) {
697  const MachineOperand &Op = IA->getOperand(I);
698  if (Op.isReg() && Op.isDef()) {
699  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
700  }
701  }
702 
703  return WaitStatesNeeded;
704 }
705 
706 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
707  const SIInstrInfo *TII = ST.getInstrInfo();
708  const SIRegisterInfo *TRI = ST.getRegisterInfo();
709  const MachineRegisterInfo &MRI = MF.getRegInfo();
710 
711  const MachineOperand *LaneSelectOp =
712  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
713 
714  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
715  return 0;
716 
717  unsigned LaneSelectReg = LaneSelectOp->getReg();
718  auto IsHazardFn = [TII] (MachineInstr *MI) {
719  return TII->isVALU(*MI);
720  };
721 
722  const int RWLaneWaitStates = 4;
723  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
724  RWLaneWaitStates);
725  return RWLaneWaitStates - WaitStatesSince;
726 }
727 
728 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
730  return 0;
731 
732  const SIInstrInfo *TII = ST.getInstrInfo();
733 
734  const int RFEWaitStates = 1;
735 
736  auto IsHazardFn = [TII] (MachineInstr *MI) {
737  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
738  };
739  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
740  return RFEWaitStates - WaitStatesNeeded;
741 }
742 
743 int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
744  if (MI->isDebugInstr())
745  return 0;
746 
747  const SIRegisterInfo *TRI = ST.getRegisterInfo();
748  if (!ST.hasSMovFedHazard())
749  return 0;
750 
751  // Check for any instruction reading an SGPR after a write from
752  // s_mov_fed_b32.
753  int MovFedWaitStates = 1;
754  int WaitStatesNeeded = 0;
755 
756  for (const MachineOperand &Use : MI->uses()) {
757  if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
758  continue;
759  auto IsHazardFn = [] (MachineInstr *MI) {
760  return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
761  };
762  int WaitStatesNeededForUse =
763  MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn,
764  MovFedWaitStates);
765  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
766  }
767 
768  return WaitStatesNeeded;
769 }
770 
771 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
772  const SIInstrInfo *TII = ST.getInstrInfo();
773  const int SMovRelWaitStates = 1;
774  auto IsHazardFn = [TII] (MachineInstr *MI) {
775  return TII->isSALU(*MI);
776  };
777  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
778  SMovRelWaitStates);
779 }
static bool isSMovRel(unsigned Opcode)
BitVector & set()
Definition: BitVector.h:397
Interface definition for SIRegisterInfo.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
AMDGPU specific subclass of TargetSubtarget.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:491
bool hasReadM0MovRelInterpHazard() const
Implements a dense probed hash-table based set.
Definition: DenseSet.h:249
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:542
unsigned Reg
bool isInlineAsm() const
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:116
static bool isRFE(unsigned Opcode)
const SIInstrInfo * getInstrInfo() const override
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, unsigned Reg)
unsigned const TargetRegisterInfo * TRI
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:434
function_ref< bool(MachineInstr *)> IsHazardFn
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:444
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:470
const HexagonInstrInfo * TII
bool isAlwaysGDS(uint16_t Opcode) const
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:411
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:454
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
bool hasSMovFedHazard() const
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:322
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:405
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:418
static bool isSSetReg(unsigned Opcode)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
unsigned PreEmitNoops(SUnit *SU) override
PreEmitNoops - This callback is invoked prior to emitting an instruction.
function_ref< bool(MachineInstr *, int WaitStates)> IsExpiredFn
reverse_iterator rend()
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:526
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:819
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
unsigned const MachineRegisterInfo * MRI
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:523
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, MachineBasicBlock *MBB, MachineBasicBlock::reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock *> &Visited)
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:480
Generation getGeneration() const
iterator_range< pred_iterator > predecessors()
bool isImplicitDef() const
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
bool isDebugInstr() const
Definition: MachineInstr.h:998
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Iterator for intrusive lists based on ilist_node.
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:314
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
MachineOperand class - Representation of each machine instruction operand.
GCNHazardRecognizer(const MachineFunction &MF)
int64_t getImm() const
A range adaptor for a pair of iterators.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
static bool isRWLane(unsigned Opcode)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:201
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
Interface definition for SIInstrInfo.
reverse_self_iterator getReverseIterator()
Definition: ilist_node.h:84
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:426
bool has12DWordStoreHazard() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:72
bool isXNACKEnabled() const
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:330
bool isReg() const
isReg - Tests if this is a MO_Register operand.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSGetReg(unsigned Opcode)
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
IRTranslator LLVM IR MI
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
bool hasReadM0SendMsgHazard() const
bool isBufferSMRD(const MachineInstr &MI) const
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
unsigned PreEmitNoopsCommon(MachineInstr *)
const SIRegisterInfo * getRegisterInfo() const override
static bool isDivFMas(unsigned Opcode)