LLVM  8.0.0svn
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements hazard recognizers for scheduling on GCN processors.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "GCNHazardRecognizer.h"
15 #include "AMDGPUSubtarget.h"
16 #include "SIDefines.h"
17 #include "SIInstrInfo.h"
18 #include "SIRegisterInfo.h"
20 #include "Utils/AMDGPUBaseInfo.h"
26 #include "llvm/MC/MCInstrDesc.h"
28 #include <algorithm>
29 #include <cassert>
30 #include <limits>
31 #include <set>
32 #include <vector>
33 
34 using namespace llvm;
35 
36 //===----------------------------------------------------------------------===//
37 // Hazard Recoginizer Implementation
38 //===----------------------------------------------------------------------===//
39 
41  CurrCycleInstr(nullptr),
42  MF(MF),
43  ST(MF.getSubtarget<GCNSubtarget>()),
44  TII(*ST.getInstrInfo()),
45  TRI(TII.getRegisterInfo()),
46  ClauseUses(TRI.getNumRegUnits()),
47  ClauseDefs(TRI.getNumRegUnits()) {
48  MaxLookAhead = 5;
49 }
50 
53 }
54 
56  CurrCycleInstr = MI;
57 }
58 
59 static bool isDivFMas(unsigned Opcode) {
60  return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
61 }
62 
63 static bool isSGetReg(unsigned Opcode) {
64  return Opcode == AMDGPU::S_GETREG_B32;
65 }
66 
67 static bool isSSetReg(unsigned Opcode) {
68  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
69 }
70 
71 static bool isRWLane(unsigned Opcode) {
72  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
73 }
74 
75 static bool isRFE(unsigned Opcode) {
76  return Opcode == AMDGPU::S_RFE_B64;
77 }
78 
79 static bool isSMovRel(unsigned Opcode) {
80  switch (Opcode) {
81  case AMDGPU::S_MOVRELS_B32:
82  case AMDGPU::S_MOVRELS_B64:
83  case AMDGPU::S_MOVRELD_B32:
84  case AMDGPU::S_MOVRELD_B64:
85  return true;
86  default:
87  return false;
88  }
89 }
90 
92  switch (MI.getOpcode()) {
93  case AMDGPU::S_SENDMSG:
94  case AMDGPU::S_SENDMSGHALT:
95  case AMDGPU::S_TTRACEDATA:
96  return true;
97  default:
98  // TODO: GDS
99  return false;
100  }
101 }
102 
103 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
104  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
105  AMDGPU::OpName::simm16);
106  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
107 }
108 
111  MachineInstr *MI = SU->getInstr();
112 
113  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
114  return NoopHazard;
115 
116  // FIXME: Should flat be considered vmem?
117  if ((SIInstrInfo::isVMEM(*MI) ||
118  SIInstrInfo::isFLAT(*MI))
119  && checkVMEMHazards(MI) > 0)
120  return NoopHazard;
121 
122  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
123  return NoopHazard;
124 
125  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
126  return NoopHazard;
127 
128  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
129  return NoopHazard;
130 
131  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
132  return NoopHazard;
133 
134  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
135  return NoopHazard;
136 
137  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
138  return NoopHazard;
139 
140  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
141  return NoopHazard;
142 
143  if (ST.hasReadM0MovRelInterpHazard() &&
144  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
145  checkReadM0Hazards(MI) > 0)
146  return NoopHazard;
147 
149  checkReadM0Hazards(MI) > 0)
150  return NoopHazard;
151 
152  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
153  return NoopHazard;
154 
155  if (checkAnyInstHazards(MI) > 0)
156  return NoopHazard;
157 
158  return NoHazard;
159 }
160 
162  return PreEmitNoops(SU->getInstr());
163 }
164 
166  int WaitStates = std::max(0, checkAnyInstHazards(MI));
167 
168  if (SIInstrInfo::isSMRD(*MI))
169  return std::max(WaitStates, checkSMRDHazards(MI));
170 
171  if (SIInstrInfo::isVALU(*MI))
172  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
173 
174  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
175  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
176 
177  if (SIInstrInfo::isDPP(*MI))
178  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
179 
180  if (isDivFMas(MI->getOpcode()))
181  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
182 
183  if (isRWLane(MI->getOpcode()))
184  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
185 
186  if (MI->isInlineAsm())
187  return std::max(WaitStates, checkInlineAsmHazards(MI));
188 
189  if (isSGetReg(MI->getOpcode()))
190  return std::max(WaitStates, checkGetRegHazards(MI));
191 
192  if (isSSetReg(MI->getOpcode()))
193  return std::max(WaitStates, checkSetRegHazards(MI));
194 
195  if (isRFE(MI->getOpcode()))
196  return std::max(WaitStates, checkRFEHazards(MI));
197 
198  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
199  isSMovRel(MI->getOpcode())))
200  return std::max(WaitStates, checkReadM0Hazards(MI));
201 
203  return std::max(WaitStates, checkReadM0Hazards(MI));
204 
205  return WaitStates;
206 }
207 
209  EmittedInstrs.push_front(nullptr);
210 }
211 
213  // When the scheduler detects a stall, it will call AdvanceCycle() without
214  // emitting any instructions.
215  if (!CurrCycleInstr)
216  return;
217 
218  // Do not track non-instructions which do not affect the wait states.
219  // If included, these instructions can lead to buffer overflow such that
220  // detectable hazards are missed.
221  if (CurrCycleInstr->getOpcode() == AMDGPU::IMPLICIT_DEF)
222  return;
223  else if (CurrCycleInstr->isDebugInstr())
224  return;
225 
226  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
227 
228  // Keep track of emitted instructions
229  EmittedInstrs.push_front(CurrCycleInstr);
230 
231  // Add a nullptr for each additional wait state after the first. Make sure
232  // not to add more than getMaxLookAhead() items to the list, since we
233  // truncate the list to that size right after this loop.
234  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
235  i < e; ++i) {
236  EmittedInstrs.push_front(nullptr);
237  }
238 
239  // getMaxLookahead() is the largest number of wait states we will ever need
240  // to insert, so there is no point in keeping track of more than that many
241  // wait states.
242  EmittedInstrs.resize(getMaxLookAhead());
243 
244  CurrCycleInstr = nullptr;
245 }
246 
248  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
249 }
250 
251 //===----------------------------------------------------------------------===//
252 // Helper Functions
253 //===----------------------------------------------------------------------===//
254 
255 int GCNHazardRecognizer::getWaitStatesSince(
256  function_ref<bool(MachineInstr *)> IsHazard) {
257  int WaitStates = 0;
258  for (MachineInstr *MI : EmittedInstrs) {
259  if (MI) {
260  if (IsHazard(MI))
261  return WaitStates;
262 
263  unsigned Opcode = MI->getOpcode();
264  if (Opcode == AMDGPU::INLINEASM)
265  continue;
266  }
267  ++WaitStates;
268  }
270 }
271 
272 int GCNHazardRecognizer::getWaitStatesSinceDef(
273  unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) {
274  const SIRegisterInfo *TRI = ST.getRegisterInfo();
275 
276  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
277  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
278  };
279 
280  return getWaitStatesSince(IsHazardFn);
281 }
282 
283 int GCNHazardRecognizer::getWaitStatesSinceSetReg(
284  function_ref<bool(MachineInstr *)> IsHazard) {
285  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
286  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
287  };
288 
289  return getWaitStatesSince(IsHazardFn);
290 }
291 
292 //===----------------------------------------------------------------------===//
293 // No-op Hazard Detection
294 //===----------------------------------------------------------------------===//
295 
296 static void addRegUnits(const SIRegisterInfo &TRI,
297  BitVector &BV, unsigned Reg) {
298  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
299  BV.set(*RUI);
300 }
301 
302 static void addRegsToSet(const SIRegisterInfo &TRI,
304  BitVector &Set) {
305  for (const MachineOperand &Op : Ops) {
306  if (Op.isReg())
307  addRegUnits(TRI, Set, Op.getReg());
308  }
309 }
310 
311 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
312  // XXX: Do we need to worry about implicit operands
313  addRegsToSet(TRI, MI.defs(), ClauseDefs);
314  addRegsToSet(TRI, MI.uses(), ClauseUses);
315 }
316 
317 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
318  // SMEM soft clause are only present on VI+, and only matter if xnack is
319  // enabled.
320  if (!ST.isXNACKEnabled())
321  return 0;
322 
323  bool IsSMRD = TII.isSMRD(*MEM);
324 
325  resetClause();
326 
327  // A soft-clause is any group of consecutive SMEM instructions. The
328  // instructions in this group may return out of order and/or may be
329  // replayed (i.e. the same instruction issued more than once).
330  //
331  // In order to handle these situations correctly we need to make sure
332  // that when a clause has more than one instruction, no instruction in the
333  // clause writes to a register that is read another instruction in the clause
334  // (including itself). If we encounter this situaion, we need to break the
335  // clause by inserting a non SMEM instruction.
336 
337  for (MachineInstr *MI : EmittedInstrs) {
338  // When we hit a non-SMEM instruction then we have passed the start of the
339  // clause and we can stop.
340  if (!MI)
341  break;
342 
343  if (IsSMRD != SIInstrInfo::isSMRD(*MI))
344  break;
345 
346  addClauseInst(*MI);
347  }
348 
349  if (ClauseDefs.none())
350  return 0;
351 
352  // We need to make sure not to put loads and stores in the same clause if they
353  // use the same address. For now, just start a new clause whenever we see a
354  // store.
355  if (MEM->mayStore())
356  return 1;
357 
358  addClauseInst(*MEM);
359 
360  // If the set of defs and uses intersect then we cannot add this instruction
361  // to the clause, so we have a hazard.
362  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
363 }
364 
365 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
366  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
367  int WaitStatesNeeded = 0;
368 
369  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
370 
371  // This SMRD hazard only affects SI.
373  return WaitStatesNeeded;
374 
375  // A read of an SGPR by SMRD instruction requires 4 wait states when the
376  // SGPR was written by a VALU instruction.
377  int SmrdSgprWaitStates = 4;
378  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
379  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
380 
381  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
382 
383  for (const MachineOperand &Use : SMRD->uses()) {
384  if (!Use.isReg())
385  continue;
386  int WaitStatesNeededForUse =
387  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
388  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
389 
390  // This fixes what appears to be undocumented hardware behavior in SI where
391  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
392  // needs some number of nops in between. We don't know how many we need, but
393  // let's use 4. This wasn't discovered before probably because the only
394  // case when this happens is when we expand a 64-bit pointer into a full
395  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
396  // probably never encountered in the closed-source land.
397  if (IsBufferSMRD) {
398  int WaitStatesNeededForUse =
399  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
400  IsBufferHazardDefFn);
401  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
402  }
403  }
404 
405  return WaitStatesNeeded;
406 }
407 
408 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
410  return 0;
411 
412  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
413 
414  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
415  // SGPR was written by a VALU Instruction.
416  const int VmemSgprWaitStates = 5;
417  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
418 
419  for (const MachineOperand &Use : VMEM->uses()) {
420  if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
421  continue;
422 
423  int WaitStatesNeededForUse =
424  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
425  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
426  }
427  return WaitStatesNeeded;
428 }
429 
430 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
431  const SIRegisterInfo *TRI = ST.getRegisterInfo();
432  const SIInstrInfo *TII = ST.getInstrInfo();
433 
434  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
435  int DppVgprWaitStates = 2;
436  int DppExecWaitStates = 5;
437  int WaitStatesNeeded = 0;
438  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
439 
440  for (const MachineOperand &Use : DPP->uses()) {
441  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
442  continue;
443  int WaitStatesNeededForUse =
444  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
445  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
446  }
447 
448  WaitStatesNeeded = std::max(
449  WaitStatesNeeded,
450  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn));
451 
452  return WaitStatesNeeded;
453 }
454 
455 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
456  const SIInstrInfo *TII = ST.getInstrInfo();
457 
458  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
459  // instruction.
460  const int DivFMasWaitStates = 4;
461  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
462  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn);
463 
464  return DivFMasWaitStates - WaitStatesNeeded;
465 }
466 
467 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
468  const SIInstrInfo *TII = ST.getInstrInfo();
469  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
470 
471  const int GetRegWaitStates = 2;
472  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
473  return GetRegHWReg == getHWReg(TII, *MI);
474  };
475  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
476 
477  return GetRegWaitStates - WaitStatesNeeded;
478 }
479 
480 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
481  const SIInstrInfo *TII = ST.getInstrInfo();
482  unsigned HWReg = getHWReg(TII, *SetRegInstr);
483 
484  const int SetRegWaitStates =
486  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
487  return HWReg == getHWReg(TII, *MI);
488  };
489  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
490  return SetRegWaitStates - WaitStatesNeeded;
491 }
492 
493 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
494  if (!MI.mayStore())
495  return -1;
496 
497  const SIInstrInfo *TII = ST.getInstrInfo();
498  unsigned Opcode = MI.getOpcode();
499  const MCInstrDesc &Desc = MI.getDesc();
500 
501  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
502  int VDataRCID = -1;
503  if (VDataIdx != -1)
504  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
505 
506  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
507  // There is no hazard if the instruction does not use vector regs
508  // (like wbinvl1)
509  if (VDataIdx == -1)
510  return -1;
511  // For MUBUF/MTBUF instructions this hazard only exists if the
512  // instruction is not using a register in the soffset field.
513  const MachineOperand *SOffset =
514  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
515  // If we have no soffset operand, then assume this field has been
516  // hardcoded to zero.
517  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
518  (!SOffset || !SOffset->isReg()))
519  return VDataIdx;
520  }
521 
522  // MIMG instructions create a hazard if they don't use a 256-bit T# and
523  // the store size is greater than 8 bytes and they have more than two bits
524  // of their dmask set.
525  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
526  if (TII->isMIMG(MI)) {
527  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
528  assert(SRsrcIdx != -1 &&
529  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
530  (void)SRsrcIdx;
531  }
532 
533  if (TII->isFLAT(MI)) {
534  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
535  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
536  return DataIdx;
537  }
538 
539  return -1;
540 }
541 
542 int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
543  const MachineRegisterInfo &MRI) {
544  // Helper to check for the hazard where VMEM instructions that store more than
545  // 8 bytes can have there store data over written by the next instruction.
546  const SIRegisterInfo *TRI = ST.getRegisterInfo();
547 
548  const int VALUWaitStates = 1;
549  int WaitStatesNeeded = 0;
550 
551  if (!TRI->isVGPR(MRI, Def.getReg()))
552  return WaitStatesNeeded;
553  unsigned Reg = Def.getReg();
554  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
555  int DataIdx = createsVALUHazard(*MI);
556  return DataIdx >= 0 &&
557  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
558  };
559  int WaitStatesNeededForDef =
560  VALUWaitStates - getWaitStatesSince(IsHazardFn);
561  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
562 
563  return WaitStatesNeeded;
564 }
565 
566 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
567  // This checks for the hazard where VMEM instructions that store more than
568  // 8 bytes can have there store data over written by the next instruction.
569  if (!ST.has12DWordStoreHazard())
570  return 0;
571 
572  const MachineRegisterInfo &MRI = MF.getRegInfo();
573  int WaitStatesNeeded = 0;
574 
575  for (const MachineOperand &Def : VALU->defs()) {
576  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
577  }
578 
579  return WaitStatesNeeded;
580 }
581 
582 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
583  // This checks for hazards associated with inline asm statements.
584  // Since inline asms can contain just about anything, we use this
585  // to call/leverage other check*Hazard routines. Note that
586  // this function doesn't attempt to address all possible inline asm
587  // hazards (good luck), but is a collection of what has been
588  // problematic thus far.
589 
590  // see checkVALUHazards()
591  if (!ST.has12DWordStoreHazard())
592  return 0;
593 
594  const MachineRegisterInfo &MRI = MF.getRegInfo();
595  int WaitStatesNeeded = 0;
596 
597  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
598  I != E; ++I) {
599  const MachineOperand &Op = IA->getOperand(I);
600  if (Op.isReg() && Op.isDef()) {
601  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
602  }
603  }
604 
605  return WaitStatesNeeded;
606 }
607 
608 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
609  const SIInstrInfo *TII = ST.getInstrInfo();
610  const SIRegisterInfo *TRI = ST.getRegisterInfo();
611  const MachineRegisterInfo &MRI = MF.getRegInfo();
612 
613  const MachineOperand *LaneSelectOp =
614  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
615 
616  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
617  return 0;
618 
619  unsigned LaneSelectReg = LaneSelectOp->getReg();
620  auto IsHazardFn = [TII] (MachineInstr *MI) {
621  return TII->isVALU(*MI);
622  };
623 
624  const int RWLaneWaitStates = 4;
625  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
626  return RWLaneWaitStates - WaitStatesSince;
627 }
628 
629 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
631  return 0;
632 
633  const SIInstrInfo *TII = ST.getInstrInfo();
634 
635  const int RFEWaitStates = 1;
636 
637  auto IsHazardFn = [TII] (MachineInstr *MI) {
638  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
639  };
640  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
641  return RFEWaitStates - WaitStatesNeeded;
642 }
643 
644 int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
645  if (MI->isDebugInstr())
646  return 0;
647 
648  const SIRegisterInfo *TRI = ST.getRegisterInfo();
649  if (!ST.hasSMovFedHazard())
650  return 0;
651 
652  // Check for any instruction reading an SGPR after a write from
653  // s_mov_fed_b32.
654  int MovFedWaitStates = 1;
655  int WaitStatesNeeded = 0;
656 
657  for (const MachineOperand &Use : MI->uses()) {
658  if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
659  continue;
660  auto IsHazardFn = [] (MachineInstr *MI) {
661  return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
662  };
663  int WaitStatesNeededForUse =
664  MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn);
665  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
666  }
667 
668  return WaitStatesNeeded;
669 }
670 
671 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
672  const SIInstrInfo *TII = ST.getInstrInfo();
673  const int SMovRelWaitStates = 1;
674  auto IsHazardFn = [TII] (MachineInstr *MI) {
675  return TII->isSALU(*MI);
676  };
677  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn);
678 }
static bool isSMovRel(unsigned Opcode)
BitVector & set()
Definition: BitVector.h:398
Interface definition for SIRegisterInfo.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
AMDGPU specific subclass of TargetSubtarget.
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:492
bool hasReadM0MovRelInterpHazard() const
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:531
static bool isSendMsgTraceDataOrGDS(const MachineInstr &MI)
unsigned Reg
bool isInlineAsm() const
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:107
static bool isRFE(unsigned Opcode)
const SIInstrInfo * getInstrInfo() const override
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, unsigned Reg)
unsigned const TargetRegisterInfo * TRI
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:425
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:459
const HexagonInstrInfo * TII
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:412
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:443
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
bool hasSMovFedHazard() const
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:409
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:313
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:645
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:406
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:409
static bool isSSetReg(unsigned Opcode)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
unsigned PreEmitNoops(SUnit *SU) override
PreEmitNoops - This callback is invoked prior to emitting an instruction.
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:515
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:820
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:377
unsigned const MachineRegisterInfo * MRI
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:524
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:481
Generation getGeneration() const
bool isDebugInstr() const
Definition: MachineInstr.h:999
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumWaitStates(const MachineInstr &MI) const
Return the number of wait states that result from executing this instruction.
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:305
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
MachineOperand class - Representation of each machine instruction operand.
GCNHazardRecognizer(const MachineFunction &MF)
int64_t getImm() const
A range adaptor for a pair of iterators.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
static bool isRWLane(unsigned Opcode)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:202
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:64
Interface definition for SIInstrInfo.
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:417
bool has12DWordStoreHazard() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:73
bool isXNACKEnabled() const
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:321
bool isReg() const
isReg - Tests if this is a MO_Register operand.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSGetReg(unsigned Opcode)
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
IRTranslator LLVM IR MI
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:414
bool hasReadM0SendMsgHazard() const
bool isBufferSMRD(const MachineInstr &MI) const
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:246
const SIRegisterInfo * getRegisterInfo() const override
static bool isDivFMas(unsigned Opcode)