LLVM  7.0.0svn
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements hazard recognizers for scheduling on GCN processors.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "GCNHazardRecognizer.h"
15 #include "AMDGPUSubtarget.h"
16 #include "SIDefines.h"
17 #include "SIInstrInfo.h"
18 #include "SIRegisterInfo.h"
19 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/MC/MCInstrDesc.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <limits>
30 #include <set>
31 #include <vector>
32 
33 using namespace llvm;
34 
35 //===----------------------------------------------------------------------===//
36 // Hazard Recoginizer Implementation
37 //===----------------------------------------------------------------------===//
38 
40  CurrCycleInstr(nullptr),
41  MF(MF),
42  ST(MF.getSubtarget<SISubtarget>()),
43  TII(*ST.getInstrInfo()),
44  TRI(TII.getRegisterInfo()),
45  ClauseUses(TRI.getNumRegUnits()),
46  ClauseDefs(TRI.getNumRegUnits()) {
47  MaxLookAhead = 5;
48 }
49 
52 }
53 
55  CurrCycleInstr = MI;
56 }
57 
58 static bool isDivFMas(unsigned Opcode) {
59  return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
60 }
61 
62 static bool isSGetReg(unsigned Opcode) {
63  return Opcode == AMDGPU::S_GETREG_B32;
64 }
65 
66 static bool isSSetReg(unsigned Opcode) {
67  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
68 }
69 
70 static bool isRWLane(unsigned Opcode) {
71  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
72 }
73 
74 static bool isRFE(unsigned Opcode) {
75  return Opcode == AMDGPU::S_RFE_B64;
76 }
77 
78 static bool isSMovRel(unsigned Opcode) {
79  switch (Opcode) {
80  case AMDGPU::S_MOVRELS_B32:
81  case AMDGPU::S_MOVRELS_B64:
82  case AMDGPU::S_MOVRELD_B32:
83  case AMDGPU::S_MOVRELD_B64:
84  return true;
85  default:
86  return false;
87  }
88 }
89 
91  switch (MI.getOpcode()) {
92  case AMDGPU::S_SENDMSG:
93  case AMDGPU::S_SENDMSGHALT:
94  case AMDGPU::S_TTRACEDATA:
95  return true;
96  default:
97  // TODO: GDS
98  return false;
99  }
100 }
101 
102 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
103  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
104  AMDGPU::OpName::simm16);
105  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
106 }
107 
110  MachineInstr *MI = SU->getInstr();
111 
112  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
113  return NoopHazard;
114 
115  // FIXME: Should flat be considered vmem?
116  if ((SIInstrInfo::isVMEM(*MI) ||
117  SIInstrInfo::isFLAT(*MI))
118  && checkVMEMHazards(MI) > 0)
119  return NoopHazard;
120 
121  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
122  return NoopHazard;
123 
124  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
125  return NoopHazard;
126 
127  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
128  return NoopHazard;
129 
130  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
131  return NoopHazard;
132 
133  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
134  return NoopHazard;
135 
136  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
137  return NoopHazard;
138 
139  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
140  return NoopHazard;
141 
142  if (ST.hasReadM0MovRelInterpHazard() &&
143  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
144  checkReadM0Hazards(MI) > 0)
145  return NoopHazard;
146 
148  checkReadM0Hazards(MI) > 0)
149  return NoopHazard;
150 
151  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
152  return NoopHazard;
153 
154  if (checkAnyInstHazards(MI) > 0)
155  return NoopHazard;
156 
157  return NoHazard;
158 }
159 
161  return PreEmitNoops(SU->getInstr());
162 }
163 
165  int WaitStates = std::max(0, checkAnyInstHazards(MI));
166 
167  if (SIInstrInfo::isSMRD(*MI))
168  return std::max(WaitStates, checkSMRDHazards(MI));
169 
170  if (SIInstrInfo::isVALU(*MI))
171  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
172 
173  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
174  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
175 
176  if (SIInstrInfo::isDPP(*MI))
177  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
178 
179  if (isDivFMas(MI->getOpcode()))
180  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
181 
182  if (isRWLane(MI->getOpcode()))
183  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
184 
185  if (MI->isInlineAsm())
186  return std::max(WaitStates, checkInlineAsmHazards(MI));
187 
188  if (isSGetReg(MI->getOpcode()))
189  return std::max(WaitStates, checkGetRegHazards(MI));
190 
191  if (isSSetReg(MI->getOpcode()))
192  return std::max(WaitStates, checkSetRegHazards(MI));
193 
194  if (isRFE(MI->getOpcode()))
195  return std::max(WaitStates, checkRFEHazards(MI));
196 
197  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
198  isSMovRel(MI->getOpcode())))
199  return std::max(WaitStates, checkReadM0Hazards(MI));
200 
202  return std::max(WaitStates, checkReadM0Hazards(MI));
203 
204  return WaitStates;
205 }
206 
208  EmittedInstrs.push_front(nullptr);
209 }
210 
212  // When the scheduler detects a stall, it will call AdvanceCycle() without
213  // emitting any instructions.
214  if (!CurrCycleInstr)
215  return;
216 
217  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
218 
219  // Keep track of emitted instructions
220  EmittedInstrs.push_front(CurrCycleInstr);
221 
222  // Add a nullptr for each additional wait state after the first. Make sure
223  // not to add more than getMaxLookAhead() items to the list, since we
224  // truncate the list to that size right after this loop.
225  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
226  i < e; ++i) {
227  EmittedInstrs.push_front(nullptr);
228  }
229 
230  // getMaxLookahead() is the largest number of wait states we will ever need
231  // to insert, so there is no point in keeping track of more than that many
232  // wait states.
233  EmittedInstrs.resize(getMaxLookAhead());
234 
235  CurrCycleInstr = nullptr;
236 }
237 
239  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
240 }
241 
242 //===----------------------------------------------------------------------===//
243 // Helper Functions
244 //===----------------------------------------------------------------------===//
245 
246 int GCNHazardRecognizer::getWaitStatesSince(
247  function_ref<bool(MachineInstr *)> IsHazard) {
248  int WaitStates = 0;
249  for (MachineInstr *MI : EmittedInstrs) {
250  if (MI) {
251  if (IsHazard(MI))
252  return WaitStates;
253 
254  unsigned Opcode = MI->getOpcode();
255  if (Opcode == AMDGPU::DBG_VALUE || Opcode == AMDGPU::IMPLICIT_DEF ||
256  Opcode == AMDGPU::INLINEASM)
257  continue;
258  }
259  ++WaitStates;
260  }
262 }
263 
264 int GCNHazardRecognizer::getWaitStatesSinceDef(
265  unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) {
266  const SIRegisterInfo *TRI = ST.getRegisterInfo();
267 
268  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
269  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
270  };
271 
272  return getWaitStatesSince(IsHazardFn);
273 }
274 
275 int GCNHazardRecognizer::getWaitStatesSinceSetReg(
276  function_ref<bool(MachineInstr *)> IsHazard) {
277  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
278  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
279  };
280 
281  return getWaitStatesSince(IsHazardFn);
282 }
283 
284 //===----------------------------------------------------------------------===//
285 // No-op Hazard Detection
286 //===----------------------------------------------------------------------===//
287 
288 static void addRegUnits(const SIRegisterInfo &TRI,
289  BitVector &BV, unsigned Reg) {
290  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
291  BV.set(*RUI);
292 }
293 
294 static void addRegsToSet(const SIRegisterInfo &TRI,
296  BitVector &Set) {
297  for (const MachineOperand &Op : Ops) {
298  if (Op.isReg())
299  addRegUnits(TRI, Set, Op.getReg());
300  }
301 }
302 
303 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
304  // XXX: Do we need to worry about implicit operands
305  addRegsToSet(TRI, MI.defs(), ClauseDefs);
306  addRegsToSet(TRI, MI.uses(), ClauseUses);
307 }
308 
309 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
310  // SMEM soft clause are only present on VI+, and only matter if xnack is
311  // enabled.
312  if (!ST.isXNACKEnabled())
313  return 0;
314 
315  bool IsSMRD = TII.isSMRD(*MEM);
316 
317  resetClause();
318 
319  // A soft-clause is any group of consecutive SMEM instructions. The
320  // instructions in this group may return out of order and/or may be
321  // replayed (i.e. the same instruction issued more than once).
322  //
323  // In order to handle these situations correctly we need to make sure
324  // that when a clause has more than one instruction, no instruction in the
325  // clause writes to a register that is read another instruction in the clause
326  // (including itself). If we encounter this situaion, we need to break the
327  // clause by inserting a non SMEM instruction.
328 
329  for (MachineInstr *MI : EmittedInstrs) {
330  // When we hit a non-SMEM instruction then we have passed the start of the
331  // clause and we can stop.
332  if (!MI)
333  break;
334 
335  if (IsSMRD != SIInstrInfo::isSMRD(*MI))
336  break;
337 
338  addClauseInst(*MI);
339  }
340 
341  if (ClauseDefs.none())
342  return 0;
343 
344  // We need to make sure not to put loads and stores in the same clause if they
345  // use the same address. For now, just start a new clause whenever we see a
346  // store.
347  if (MEM->mayStore())
348  return 1;
349 
350  addClauseInst(*MEM);
351 
352  // If the set of defs and uses intersect then we cannot add this instruction
353  // to the clause, so we have a hazard.
354  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
355 }
356 
357 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
358  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
359  int WaitStatesNeeded = 0;
360 
361  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
362 
363  // This SMRD hazard only affects SI.
365  return WaitStatesNeeded;
366 
367  // A read of an SGPR by SMRD instruction requires 4 wait states when the
368  // SGPR was written by a VALU instruction.
369  int SmrdSgprWaitStates = 4;
370  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
371  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
372 
373  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
374 
375  for (const MachineOperand &Use : SMRD->uses()) {
376  if (!Use.isReg())
377  continue;
378  int WaitStatesNeededForUse =
379  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
380  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
381 
382  // This fixes what appears to be undocumented hardware behavior in SI where
383  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
384  // needs some number of nops in between. We don't know how many we need, but
385  // let's use 4. This wasn't discovered before probably because the only
386  // case when this happens is when we expand a 64-bit pointer into a full
387  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
388  // probably never encountered in the closed-source land.
389  if (IsBufferSMRD) {
390  int WaitStatesNeededForUse =
391  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
392  IsBufferHazardDefFn);
393  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
394  }
395  }
396 
397  return WaitStatesNeeded;
398 }
399 
400 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
402  return 0;
403 
404  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
405 
406  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
407  // SGPR was written by a VALU Instruction.
408  const int VmemSgprWaitStates = 5;
409  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
410 
411  for (const MachineOperand &Use : VMEM->uses()) {
412  if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
413  continue;
414 
415  int WaitStatesNeededForUse =
416  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
417  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
418  }
419  return WaitStatesNeeded;
420 }
421 
422 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
423  const SIRegisterInfo *TRI = ST.getRegisterInfo();
424  const SIInstrInfo *TII = ST.getInstrInfo();
425 
426  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
427  int DppVgprWaitStates = 2;
428  int DppExecWaitStates = 5;
429  int WaitStatesNeeded = 0;
430  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
431 
432  for (const MachineOperand &Use : DPP->uses()) {
433  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
434  continue;
435  int WaitStatesNeededForUse =
436  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
437  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
438  }
439 
440  WaitStatesNeeded = std::max(
441  WaitStatesNeeded,
442  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn));
443 
444  return WaitStatesNeeded;
445 }
446 
447 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
448  const SIInstrInfo *TII = ST.getInstrInfo();
449 
450  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
451  // instruction.
452  const int DivFMasWaitStates = 4;
453  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
454  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn);
455 
456  return DivFMasWaitStates - WaitStatesNeeded;
457 }
458 
459 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
460  const SIInstrInfo *TII = ST.getInstrInfo();
461  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
462 
463  const int GetRegWaitStates = 2;
464  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
465  return GetRegHWReg == getHWReg(TII, *MI);
466  };
467  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
468 
469  return GetRegWaitStates - WaitStatesNeeded;
470 }
471 
472 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
473  const SIInstrInfo *TII = ST.getInstrInfo();
474  unsigned HWReg = getHWReg(TII, *SetRegInstr);
475 
476  const int SetRegWaitStates =
478  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
479  return HWReg == getHWReg(TII, *MI);
480  };
481  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
482  return SetRegWaitStates - WaitStatesNeeded;
483 }
484 
485 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
486  if (!MI.mayStore())
487  return -1;
488 
489  const SIInstrInfo *TII = ST.getInstrInfo();
490  unsigned Opcode = MI.getOpcode();
491  const MCInstrDesc &Desc = MI.getDesc();
492 
493  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
494  int VDataRCID = -1;
495  if (VDataIdx != -1)
496  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
497 
498  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
499  // There is no hazard if the instruction does not use vector regs
500  // (like wbinvl1)
501  if (VDataIdx == -1)
502  return -1;
503  // For MUBUF/MTBUF instructions this hazard only exists if the
504  // instruction is not using a register in the soffset field.
505  const MachineOperand *SOffset =
506  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
507  // If we have no soffset operand, then assume this field has been
508  // hardcoded to zero.
509  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
510  (!SOffset || !SOffset->isReg()))
511  return VDataIdx;
512  }
513 
514  // MIMG instructions create a hazard if they don't use a 256-bit T# and
515  // the store size is greater than 8 bytes and they have more than two bits
516  // of their dmask set.
517  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
518  if (TII->isMIMG(MI)) {
519  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
520  assert(SRsrcIdx != -1 &&
521  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
522  (void)SRsrcIdx;
523  }
524 
525  if (TII->isFLAT(MI)) {
526  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
527  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
528  return DataIdx;
529  }
530 
531  return -1;
532 }
533 
534 int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
535  const MachineRegisterInfo &MRI) {
536  // Helper to check for the hazard where VMEM instructions that store more than
537  // 8 bytes can have there store data over written by the next instruction.
538  const SIRegisterInfo *TRI = ST.getRegisterInfo();
539 
540  const int VALUWaitStates = 1;
541  int WaitStatesNeeded = 0;
542 
543  if (!TRI->isVGPR(MRI, Def.getReg()))
544  return WaitStatesNeeded;
545  unsigned Reg = Def.getReg();
546  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
547  int DataIdx = createsVALUHazard(*MI);
548  return DataIdx >= 0 &&
549  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
550  };
551  int WaitStatesNeededForDef =
552  VALUWaitStates - getWaitStatesSince(IsHazardFn);
553  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
554 
555  return WaitStatesNeeded;
556 }
557 
558 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
559  // This checks for the hazard where VMEM instructions that store more than
560  // 8 bytes can have there store data over written by the next instruction.
561  if (!ST.has12DWordStoreHazard())
562  return 0;
563 
564  const MachineRegisterInfo &MRI = MF.getRegInfo();
565  int WaitStatesNeeded = 0;
566 
567  for (const MachineOperand &Def : VALU->defs()) {
568  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
569  }
570 
571  return WaitStatesNeeded;
572 }
573 
574 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
575  // This checks for hazards associated with inline asm statements.
576  // Since inline asms can contain just about anything, we use this
577  // to call/leverage other check*Hazard routines. Note that
578  // this function doesn't attempt to address all possible inline asm
579  // hazards (good luck), but is a collection of what has been
580  // problematic thus far.
581 
582  // see checkVALUHazards()
583  if (!ST.has12DWordStoreHazard())
584  return 0;
585 
586  const MachineRegisterInfo &MRI = MF.getRegInfo();
587  int WaitStatesNeeded = 0;
588 
589  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
590  I != E; ++I) {
591  const MachineOperand &Op = IA->getOperand(I);
592  if (Op.isReg() && Op.isDef()) {
593  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
594  }
595  }
596 
597  return WaitStatesNeeded;
598 }
599 
600 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
601  const SIInstrInfo *TII = ST.getInstrInfo();
602  const SIRegisterInfo *TRI = ST.getRegisterInfo();
603  const MachineRegisterInfo &MRI = MF.getRegInfo();
604 
605  const MachineOperand *LaneSelectOp =
606  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
607 
608  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
609  return 0;
610 
611  unsigned LaneSelectReg = LaneSelectOp->getReg();
612  auto IsHazardFn = [TII] (MachineInstr *MI) {
613  return TII->isVALU(*MI);
614  };
615 
616  const int RWLaneWaitStates = 4;
617  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
618  return RWLaneWaitStates - WaitStatesSince;
619 }
620 
621 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
623  return 0;
624 
625  const SIInstrInfo *TII = ST.getInstrInfo();
626 
627  const int RFEWaitStates = 1;
628 
629  auto IsHazardFn = [TII] (MachineInstr *MI) {
630  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
631  };
632  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
633  return RFEWaitStates - WaitStatesNeeded;
634 }
635 
636 int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
637  if (MI->isDebugValue())
638  return 0;
639 
640  const SIRegisterInfo *TRI = ST.getRegisterInfo();
641  if (!ST.hasSMovFedHazard())
642  return 0;
643 
644  // Check for any instruction reading an SGPR after a write from
645  // s_mov_fed_b32.
646  int MovFedWaitStates = 1;
647  int WaitStatesNeeded = 0;
648 
649  for (const MachineOperand &Use : MI->uses()) {
650  if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
651  continue;
652  auto IsHazardFn = [] (MachineInstr *MI) {
653  return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
654  };
655  int WaitStatesNeededForUse =
656  MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn);
657  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
658  }
659 
660  return WaitStatesNeeded;
661 }
662 
663 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
664  const SIInstrInfo *TII = ST.getInstrInfo();
665  const int SMovRelWaitStates = 1;
666  auto IsHazardFn = [TII] (MachineInstr *MI) {
667  return TII->isSALU(*MI);
668  };
669  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn);
670 }
static bool isSMovRel(unsigned Opcode)
BitVector & set()
Definition: BitVector.h:398
Interface definition for SIRegisterInfo.
Generation getGeneration() const
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
AMDGPU specific subclass of TargetSubtarget.
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:368
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:539
static bool isSendMsgTraceDataOrGDS(const MachineInstr &MI)
const SIInstrInfo * getInstrInfo() const override
bool isInlineAsm() const
Definition: MachineInstr.h:835
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:89
static bool isRFE(unsigned Opcode)
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, unsigned Reg)
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:414
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:467
const HexagonInstrInfo * TII
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:296
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:443
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:293
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:302
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:635
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:290
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:398
static bool isSSetReg(unsigned Opcode)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
unsigned PreEmitNoops(SUnit *SU) override
PreEmitNoops - This callback is invoked prior to emitting an instruction.
bool hasSMovFedHazard() const
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:523
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:642
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:378
unsigned const MachineRegisterInfo * MRI
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:507
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:357
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumWaitStates(const MachineInstr &MI) const
Return the number of wait states that result from executing this instruction.
bool has12DWordStoreHazard() const
const SIRegisterInfo * getRegisterInfo() const override
bool isDebugValue() const
Definition: MachineInstr.h:819
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:294
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
MachineOperand class - Representation of each machine instruction operand.
GCNHazardRecognizer(const MachineFunction &MF)
int64_t getImm() const
A range adaptor for a pair of iterators.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
static bool isRWLane(unsigned Opcode)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:202
Representation of each machine instruction.
Definition: MachineInstr.h:60
Interface definition for SIInstrInfo.
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:406
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:76
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:310
bool hasReadM0SendMsgHazard() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isBufferSMRD(const MachineInstr &MI) const
Definition: SIInstrInfo.h:422
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSGetReg(unsigned Opcode)
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
IRTranslator LLVM IR MI
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:298
bool isXNACKEnabled() const
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:247
bool hasReadM0MovRelInterpHazard() const
static bool isDivFMas(unsigned Opcode)