LLVM  9.0.0svn
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "AMDGPUSubtarget.h"
15 #include "SIDefines.h"
16 #include "SIInstrInfo.h"
17 #include "SIRegisterInfo.h"
19 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/MC/MCInstrDesc.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <limits>
30 #include <set>
31 #include <vector>
32 
33 using namespace llvm;
34 
35 //===----------------------------------------------------------------------===//
36 // Hazard Recoginizer Implementation
37 //===----------------------------------------------------------------------===//
38 
40  IsHazardRecognizerMode(false),
41  CurrCycleInstr(nullptr),
42  MF(MF),
43  ST(MF.getSubtarget<GCNSubtarget>()),
44  TII(*ST.getInstrInfo()),
45  TRI(TII.getRegisterInfo()),
46  ClauseUses(TRI.getNumRegUnits()),
47  ClauseDefs(TRI.getNumRegUnits()) {
48  MaxLookAhead = 5;
49 }
50 
53 }
54 
56  CurrCycleInstr = MI;
57 }
58 
59 static bool isDivFMas(unsigned Opcode) {
60  return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
61 }
62 
63 static bool isSGetReg(unsigned Opcode) {
64  return Opcode == AMDGPU::S_GETREG_B32;
65 }
66 
67 static bool isSSetReg(unsigned Opcode) {
68  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
69 }
70 
71 static bool isRWLane(unsigned Opcode) {
72  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
73 }
74 
75 static bool isRFE(unsigned Opcode) {
76  return Opcode == AMDGPU::S_RFE_B64;
77 }
78 
79 static bool isSMovRel(unsigned Opcode) {
80  switch (Opcode) {
81  case AMDGPU::S_MOVRELS_B32:
82  case AMDGPU::S_MOVRELS_B64:
83  case AMDGPU::S_MOVRELD_B32:
84  case AMDGPU::S_MOVRELD_B64:
85  return true;
86  default:
87  return false;
88  }
89 }
90 
91 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
92  const MachineInstr &MI) {
93  if (TII.isAlwaysGDS(MI.getOpcode()))
94  return true;
95 
96  switch (MI.getOpcode()) {
97  case AMDGPU::S_SENDMSG:
98  case AMDGPU::S_SENDMSGHALT:
99  case AMDGPU::S_TTRACEDATA:
100  return true;
101  // These DS opcodes don't support GDS.
102  case AMDGPU::DS_NOP:
103  case AMDGPU::DS_PERMUTE_B32:
104  case AMDGPU::DS_BPERMUTE_B32:
105  return false;
106  default:
107  if (TII.isDS(MI.getOpcode())) {
108  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
109  AMDGPU::OpName::gds);
110  if (MI.getOperand(GDS).getImm())
111  return true;
112  }
113  return false;
114  }
115 }
116 
117 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
118  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
119  AMDGPU::OpName::simm16);
120  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
121 }
122 
125  MachineInstr *MI = SU->getInstr();
126 
127  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
128  return NoopHazard;
129 
130  // FIXME: Should flat be considered vmem?
131  if ((SIInstrInfo::isVMEM(*MI) ||
132  SIInstrInfo::isFLAT(*MI))
133  && checkVMEMHazards(MI) > 0)
134  return NoopHazard;
135 
136  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
137  return NoopHazard;
138 
139  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
140  return NoopHazard;
141 
142  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
143  return NoopHazard;
144 
145  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
146  return NoopHazard;
147 
148  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
149  return NoopHazard;
150 
151  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
152  return NoopHazard;
153 
154  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
155  return NoopHazard;
156 
157  if (ST.hasReadM0MovRelInterpHazard() &&
158  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
159  checkReadM0Hazards(MI) > 0)
160  return NoopHazard;
161 
162  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
163  checkReadM0Hazards(MI) > 0)
164  return NoopHazard;
165 
166  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
167  return NoopHazard;
168 
169  if (checkAnyInstHazards(MI) > 0)
170  return NoopHazard;
171 
172  return NoHazard;
173 }
174 
176  IsHazardRecognizerMode = false;
177  return PreEmitNoopsCommon(SU->getInstr());
178 }
179 
181  IsHazardRecognizerMode = true;
182  CurrCycleInstr = MI;
183  unsigned W = PreEmitNoopsCommon(MI);
184  CurrCycleInstr = nullptr;
185  return W;
186 }
187 
189  int WaitStates = std::max(0, checkAnyInstHazards(MI));
190 
191  if (SIInstrInfo::isSMRD(*MI))
192  return std::max(WaitStates, checkSMRDHazards(MI));
193 
194  if (SIInstrInfo::isVALU(*MI))
195  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
196 
197  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
198  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
199 
200  if (SIInstrInfo::isDPP(*MI))
201  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
202 
203  if (isDivFMas(MI->getOpcode()))
204  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
205 
206  if (isRWLane(MI->getOpcode()))
207  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
208 
209  if (MI->isInlineAsm())
210  return std::max(WaitStates, checkInlineAsmHazards(MI));
211 
212  if (isSGetReg(MI->getOpcode()))
213  return std::max(WaitStates, checkGetRegHazards(MI));
214 
215  if (isSSetReg(MI->getOpcode()))
216  return std::max(WaitStates, checkSetRegHazards(MI));
217 
218  if (isRFE(MI->getOpcode()))
219  return std::max(WaitStates, checkRFEHazards(MI));
220 
221  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
222  isSMovRel(MI->getOpcode())))
223  return std::max(WaitStates, checkReadM0Hazards(MI));
224 
225  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
226  return std::max(WaitStates, checkReadM0Hazards(MI));
227 
228  return WaitStates;
229 }
230 
232  EmittedInstrs.push_front(nullptr);
233 }
234 
236  // When the scheduler detects a stall, it will call AdvanceCycle() without
237  // emitting any instructions.
238  if (!CurrCycleInstr)
239  return;
240 
241  // Do not track non-instructions which do not affect the wait states.
242  // If included, these instructions can lead to buffer overflow such that
243  // detectable hazards are missed.
244  if (CurrCycleInstr->isImplicitDef() || CurrCycleInstr->isDebugInstr() ||
245  CurrCycleInstr->isKill())
246  return;
247 
248  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
249 
250  // Keep track of emitted instructions
251  EmittedInstrs.push_front(CurrCycleInstr);
252 
253  // Add a nullptr for each additional wait state after the first. Make sure
254  // not to add more than getMaxLookAhead() items to the list, since we
255  // truncate the list to that size right after this loop.
256  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
257  i < e; ++i) {
258  EmittedInstrs.push_front(nullptr);
259  }
260 
261  // getMaxLookahead() is the largest number of wait states we will ever need
262  // to insert, so there is no point in keeping track of more than that many
263  // wait states.
264  EmittedInstrs.resize(getMaxLookAhead());
265 
266  CurrCycleInstr = nullptr;
267 }
268 
270  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
271 }
272 
273 //===----------------------------------------------------------------------===//
274 // Helper Functions
275 //===----------------------------------------------------------------------===//
276 
278 
279 // Returns a minimum wait states since \p I walking all predecessors.
280 // Only scans until \p IsExpired does not return true.
281 // Can only be run in a hazard recognizer mode.
282 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
283  MachineBasicBlock *MBB,
285  int WaitStates,
286  IsExpiredFn IsExpired,
288 
289  for (auto E = MBB->rend() ; I != E; ++I) {
290  if (IsHazard(&*I))
291  return WaitStates;
292 
293  if (I->isInlineAsm() || I->isImplicitDef() || I->isDebugInstr())
294  continue;
295 
296  WaitStates += SIInstrInfo::getNumWaitStates(*I);
297 
298  if (IsExpired(&*I, WaitStates))
300  }
301 
302  int MinWaitStates = WaitStates;
303  bool Found = false;
304  for (MachineBasicBlock *Pred : MBB->predecessors()) {
305  if (!Visited.insert(Pred).second)
306  continue;
307 
308  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
309  WaitStates, IsExpired, Visited);
310 
312  continue;
313 
314  MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
315  if (IsExpired(nullptr, MinWaitStates))
316  return MinWaitStates;
317 
318  Found = true;
319  }
320 
321  if (Found)
322  return MinWaitStates;
323 
325 }
326 
327 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
328  MachineInstr *MI,
329  IsExpiredFn IsExpired) {
331  return getWaitStatesSince(IsHazard, MI->getParent(),
332  std::next(MI->getReverseIterator()),
333  0, IsExpired, Visited);
334 }
335 
336 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
337  if (IsHazardRecognizerMode) {
338  auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
339  return WaitStates >= Limit;
340  };
341  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
342  }
343 
344  int WaitStates = 0;
345  for (MachineInstr *MI : EmittedInstrs) {
346  if (MI) {
347  if (IsHazard(MI))
348  return WaitStates;
349 
350  if (MI->isInlineAsm())
351  continue;
352  }
353  ++WaitStates;
354 
355  if (WaitStates >= Limit)
356  break;
357  }
359 }
360 
361 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
362  IsHazardFn IsHazardDef,
363  int Limit) {
364  const SIRegisterInfo *TRI = ST.getRegisterInfo();
365 
366  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
367  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
368  };
369 
370  return getWaitStatesSince(IsHazardFn, Limit);
371 }
372 
373 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
374  int Limit) {
375  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
376  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
377  };
378 
379  return getWaitStatesSince(IsHazardFn, Limit);
380 }
381 
382 //===----------------------------------------------------------------------===//
383 // No-op Hazard Detection
384 //===----------------------------------------------------------------------===//
385 
386 static void addRegUnits(const SIRegisterInfo &TRI,
387  BitVector &BV, unsigned Reg) {
388  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
389  BV.set(*RUI);
390 }
391 
392 static void addRegsToSet(const SIRegisterInfo &TRI,
394  BitVector &Set) {
395  for (const MachineOperand &Op : Ops) {
396  if (Op.isReg())
397  addRegUnits(TRI, Set, Op.getReg());
398  }
399 }
400 
401 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
402  // XXX: Do we need to worry about implicit operands
403  addRegsToSet(TRI, MI.defs(), ClauseDefs);
404  addRegsToSet(TRI, MI.uses(), ClauseUses);
405 }
406 
407 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
408  // SMEM soft clause are only present on VI+, and only matter if xnack is
409  // enabled.
410  if (!ST.isXNACKEnabled())
411  return 0;
412 
413  bool IsSMRD = TII.isSMRD(*MEM);
414 
415  resetClause();
416 
417  // A soft-clause is any group of consecutive SMEM instructions. The
418  // instructions in this group may return out of order and/or may be
419  // replayed (i.e. the same instruction issued more than once).
420  //
421  // In order to handle these situations correctly we need to make sure
422  // that when a clause has more than one instruction, no instruction in the
423  // clause writes to a register that is read another instruction in the clause
424  // (including itself). If we encounter this situaion, we need to break the
425  // clause by inserting a non SMEM instruction.
426 
427  for (MachineInstr *MI : EmittedInstrs) {
428  // When we hit a non-SMEM instruction then we have passed the start of the
429  // clause and we can stop.
430  if (!MI)
431  break;
432 
433  if (IsSMRD != SIInstrInfo::isSMRD(*MI))
434  break;
435 
436  addClauseInst(*MI);
437  }
438 
439  if (ClauseDefs.none())
440  return 0;
441 
442  // We need to make sure not to put loads and stores in the same clause if they
443  // use the same address. For now, just start a new clause whenever we see a
444  // store.
445  if (MEM->mayStore())
446  return 1;
447 
448  addClauseInst(*MEM);
449 
450  // If the set of defs and uses intersect then we cannot add this instruction
451  // to the clause, so we have a hazard.
452  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
453 }
454 
455 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
456  int WaitStatesNeeded = 0;
457 
458  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
459 
460  // This SMRD hazard only affects SI.
462  return WaitStatesNeeded;
463 
464  // A read of an SGPR by SMRD instruction requires 4 wait states when the
465  // SGPR was written by a VALU instruction.
466  int SmrdSgprWaitStates = 4;
467  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
468  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
469 
470  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
471 
472  for (const MachineOperand &Use : SMRD->uses()) {
473  if (!Use.isReg())
474  continue;
475  int WaitStatesNeededForUse =
476  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
477  SmrdSgprWaitStates);
478  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
479 
480  // This fixes what appears to be undocumented hardware behavior in SI where
481  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
482  // needs some number of nops in between. We don't know how many we need, but
483  // let's use 4. This wasn't discovered before probably because the only
484  // case when this happens is when we expand a 64-bit pointer into a full
485  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
486  // probably never encountered in the closed-source land.
487  if (IsBufferSMRD) {
488  int WaitStatesNeededForUse =
489  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
490  IsBufferHazardDefFn,
491  SmrdSgprWaitStates);
492  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
493  }
494  }
495 
496  return WaitStatesNeeded;
497 }
498 
499 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
501  return 0;
502 
503  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
504 
505  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
506  // SGPR was written by a VALU Instruction.
507  const int VmemSgprWaitStates = 5;
508  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
509 
510  for (const MachineOperand &Use : VMEM->uses()) {
511  if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
512  continue;
513 
514  int WaitStatesNeededForUse =
515  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
516  VmemSgprWaitStates);
517  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
518  }
519  return WaitStatesNeeded;
520 }
521 
522 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
523  const SIRegisterInfo *TRI = ST.getRegisterInfo();
524  const SIInstrInfo *TII = ST.getInstrInfo();
525 
526  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
527  int DppVgprWaitStates = 2;
528  int DppExecWaitStates = 5;
529  int WaitStatesNeeded = 0;
530  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
531 
532  for (const MachineOperand &Use : DPP->uses()) {
533  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
534  continue;
535  int WaitStatesNeededForUse =
536  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
537  [](MachineInstr *) { return true; },
538  DppVgprWaitStates);
539  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
540  }
541 
542  WaitStatesNeeded = std::max(
543  WaitStatesNeeded,
544  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
545  DppExecWaitStates));
546 
547  return WaitStatesNeeded;
548 }
549 
550 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
551  const SIInstrInfo *TII = ST.getInstrInfo();
552 
553  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
554  // instruction.
555  const int DivFMasWaitStates = 4;
556  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
557  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
558  DivFMasWaitStates);
559 
560  return DivFMasWaitStates - WaitStatesNeeded;
561 }
562 
563 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
564  const SIInstrInfo *TII = ST.getInstrInfo();
565  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
566 
567  const int GetRegWaitStates = 2;
568  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
569  return GetRegHWReg == getHWReg(TII, *MI);
570  };
571  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
572 
573  return GetRegWaitStates - WaitStatesNeeded;
574 }
575 
576 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
577  const SIInstrInfo *TII = ST.getInstrInfo();
578  unsigned HWReg = getHWReg(TII, *SetRegInstr);
579 
580  const int SetRegWaitStates =
582  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
583  return HWReg == getHWReg(TII, *MI);
584  };
585  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
586  return SetRegWaitStates - WaitStatesNeeded;
587 }
588 
589 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
590  if (!MI.mayStore())
591  return -1;
592 
593  const SIInstrInfo *TII = ST.getInstrInfo();
594  unsigned Opcode = MI.getOpcode();
595  const MCInstrDesc &Desc = MI.getDesc();
596 
597  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
598  int VDataRCID = -1;
599  if (VDataIdx != -1)
600  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
601 
602  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
603  // There is no hazard if the instruction does not use vector regs
604  // (like wbinvl1)
605  if (VDataIdx == -1)
606  return -1;
607  // For MUBUF/MTBUF instructions this hazard only exists if the
608  // instruction is not using a register in the soffset field.
609  const MachineOperand *SOffset =
610  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
611  // If we have no soffset operand, then assume this field has been
612  // hardcoded to zero.
613  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
614  (!SOffset || !SOffset->isReg()))
615  return VDataIdx;
616  }
617 
618  // MIMG instructions create a hazard if they don't use a 256-bit T# and
619  // the store size is greater than 8 bytes and they have more than two bits
620  // of their dmask set.
621  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
622  if (TII->isMIMG(MI)) {
623  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
624  assert(SRsrcIdx != -1 &&
625  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
626  (void)SRsrcIdx;
627  }
628 
629  if (TII->isFLAT(MI)) {
630  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
631  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
632  return DataIdx;
633  }
634 
635  return -1;
636 }
637 
638 int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
639  const MachineRegisterInfo &MRI) {
640  // Helper to check for the hazard where VMEM instructions that store more than
641  // 8 bytes can have there store data over written by the next instruction.
642  const SIRegisterInfo *TRI = ST.getRegisterInfo();
643 
644  const int VALUWaitStates = 1;
645  int WaitStatesNeeded = 0;
646 
647  if (!TRI->isVGPR(MRI, Def.getReg()))
648  return WaitStatesNeeded;
649  unsigned Reg = Def.getReg();
650  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
651  int DataIdx = createsVALUHazard(*MI);
652  return DataIdx >= 0 &&
653  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
654  };
655  int WaitStatesNeededForDef =
656  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
657  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
658 
659  return WaitStatesNeeded;
660 }
661 
662 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
663  // This checks for the hazard where VMEM instructions that store more than
664  // 8 bytes can have there store data over written by the next instruction.
665  if (!ST.has12DWordStoreHazard())
666  return 0;
667 
668  const MachineRegisterInfo &MRI = MF.getRegInfo();
669  int WaitStatesNeeded = 0;
670 
671  for (const MachineOperand &Def : VALU->defs()) {
672  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
673  }
674 
675  return WaitStatesNeeded;
676 }
677 
678 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
679  // This checks for hazards associated with inline asm statements.
680  // Since inline asms can contain just about anything, we use this
681  // to call/leverage other check*Hazard routines. Note that
682  // this function doesn't attempt to address all possible inline asm
683  // hazards (good luck), but is a collection of what has been
684  // problematic thus far.
685 
686  // see checkVALUHazards()
687  if (!ST.has12DWordStoreHazard())
688  return 0;
689 
690  const MachineRegisterInfo &MRI = MF.getRegInfo();
691  int WaitStatesNeeded = 0;
692 
693  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
694  I != E; ++I) {
695  const MachineOperand &Op = IA->getOperand(I);
696  if (Op.isReg() && Op.isDef()) {
697  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
698  }
699  }
700 
701  return WaitStatesNeeded;
702 }
703 
704 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
705  const SIInstrInfo *TII = ST.getInstrInfo();
706  const SIRegisterInfo *TRI = ST.getRegisterInfo();
707  const MachineRegisterInfo &MRI = MF.getRegInfo();
708 
709  const MachineOperand *LaneSelectOp =
710  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
711 
712  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
713  return 0;
714 
715  unsigned LaneSelectReg = LaneSelectOp->getReg();
716  auto IsHazardFn = [TII] (MachineInstr *MI) {
717  return TII->isVALU(*MI);
718  };
719 
720  const int RWLaneWaitStates = 4;
721  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
722  RWLaneWaitStates);
723  return RWLaneWaitStates - WaitStatesSince;
724 }
725 
726 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
728  return 0;
729 
730  const SIInstrInfo *TII = ST.getInstrInfo();
731 
732  const int RFEWaitStates = 1;
733 
734  auto IsHazardFn = [TII] (MachineInstr *MI) {
735  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
736  };
737  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
738  return RFEWaitStates - WaitStatesNeeded;
739 }
740 
741 int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
742  if (MI->isDebugInstr())
743  return 0;
744 
745  const SIRegisterInfo *TRI = ST.getRegisterInfo();
746  if (!ST.hasSMovFedHazard())
747  return 0;
748 
749  // Check for any instruction reading an SGPR after a write from
750  // s_mov_fed_b32.
751  int MovFedWaitStates = 1;
752  int WaitStatesNeeded = 0;
753 
754  for (const MachineOperand &Use : MI->uses()) {
755  if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
756  continue;
757  auto IsHazardFn = [] (MachineInstr *MI) {
758  return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
759  };
760  int WaitStatesNeededForUse =
761  MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn,
762  MovFedWaitStates);
763  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
764  }
765 
766  return WaitStatesNeeded;
767 }
768 
769 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
770  const SIInstrInfo *TII = ST.getInstrInfo();
771  const int SMovRelWaitStates = 1;
772  auto IsHazardFn = [TII] (MachineInstr *MI) {
773  return TII->isSALU(*MI);
774  };
775  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
776  SMovRelWaitStates);
777 }
static bool isSMovRel(unsigned Opcode)
BitVector & set()
Definition: BitVector.h:397
Interface definition for SIRegisterInfo.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
AMDGPU specific subclass of TargetSubtarget.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:491
bool hasReadM0MovRelInterpHazard() const
Implements a dense probed hash-table based set.
Definition: DenseSet.h:249
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:546
unsigned Reg
bool isInlineAsm() const
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:116
static bool isRFE(unsigned Opcode)
const SIInstrInfo * getInstrInfo() const override
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, unsigned Reg)
unsigned const TargetRegisterInfo * TRI
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:438
function_ref< bool(MachineInstr *)> IsHazardFn
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:448
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:474
const HexagonInstrInfo * TII
bool isAlwaysGDS(uint16_t Opcode) const
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:411
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:458
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
bool hasSMovFedHazard() const
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:326
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:405
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:422
static bool isSSetReg(unsigned Opcode)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
unsigned PreEmitNoops(SUnit *SU) override
PreEmitNoops - This callback is invoked prior to emitting an instruction.
function_ref< bool(MachineInstr *, int WaitStates)> IsExpiredFn
reverse_iterator rend()
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:530
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:819
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
unsigned const MachineRegisterInfo * MRI
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:523
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, MachineBasicBlock *MBB, MachineBasicBlock::reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock *> &Visited)
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:480
Generation getGeneration() const
iterator_range< pred_iterator > predecessors()
bool isImplicitDef() const
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
bool isDebugInstr() const
Definition: MachineInstr.h:998
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Iterator for intrusive lists based on ilist_node.
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:318
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
MachineOperand class - Representation of each machine instruction operand.
GCNHazardRecognizer(const MachineFunction &MF)
int64_t getImm() const
A range adaptor for a pair of iterators.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
static bool isRWLane(unsigned Opcode)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:201
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
Interface definition for SIInstrInfo.
reverse_self_iterator getReverseIterator()
Definition: ilist_node.h:84
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:430
bool has12DWordStoreHazard() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:72
bool isXNACKEnabled() const
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:334
bool isKill() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSGetReg(unsigned Opcode)
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
IRTranslator LLVM IR MI
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
bool hasReadM0SendMsgHazard() const
bool isBufferSMRD(const MachineInstr &MI) const
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
unsigned PreEmitNoopsCommon(MachineInstr *)
const SIRegisterInfo * getRegisterInfo() const override
static bool isDivFMas(unsigned Opcode)