LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - GCNHazardRecognizer.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 250 255 98.0 %
Date: 2017-09-14 15:23:50 Functions: 27 28 96.4 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements hazard recognizers for scheduling on GCN processors.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "GCNHazardRecognizer.h"
      15             : #include "AMDGPUSubtarget.h"
      16             : #include "SIDefines.h"
      17             : #include "SIInstrInfo.h"
      18             : #include "SIRegisterInfo.h"
      19             : #include "Utils/AMDGPUBaseInfo.h"
      20             : #include "llvm/ADT/iterator_range.h"
      21             : #include "llvm/CodeGen/MachineFunction.h"
      22             : #include "llvm/CodeGen/MachineInstr.h"
      23             : #include "llvm/CodeGen/MachineOperand.h"
      24             : #include "llvm/CodeGen/ScheduleDAG.h"
      25             : #include "llvm/MC/MCInstrDesc.h"
      26             : #include "llvm/Support/ErrorHandling.h"
      27             : #include <algorithm>
      28             : #include <cassert>
      29             : #include <limits>
      30             : #include <set>
      31             : #include <vector>
      32             : 
      33             : using namespace llvm;
      34             : 
      35             : //===----------------------------------------------------------------------===//
      36             : // Hazard Recoginizer Implementation
      37             : //===----------------------------------------------------------------------===//
      38             : 
      39       26138 : GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
      40             :   CurrCycleInstr(nullptr),
      41             :   MF(MF),
      42       26138 :   ST(MF.getSubtarget<SISubtarget>()),
      43      130690 :   TII(*ST.getInstrInfo()) {
      44       26138 :   MaxLookAhead = 5;
      45       26138 : }
      46             : 
      47      174965 : void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
      48      174965 :   EmitInstruction(SU->getInstr());
      49      174965 : }
      50             : 
      51      441989 : void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
      52      441989 :   CurrCycleInstr = MI;
      53      441989 : }
      54             : 
      55             : static bool isDivFMas(unsigned Opcode) {
      56      367769 :   return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
      57             : }
      58             : 
      59             : static bool isSGetReg(unsigned Opcode) {
      60             :   return Opcode == AMDGPU::S_GETREG_B32;
      61             : }
      62             : 
      63             : static bool isSSetReg(unsigned Opcode) {
      64      373635 :   return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
      65             : }
      66             : 
      67             : static bool isRWLane(unsigned Opcode) {
      68      367752 :   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
      69             : }
      70             : 
      71             : static bool isRFE(unsigned Opcode) {
      72             :   return Opcode == AMDGPU::S_RFE_B64;
      73             : }
      74             : 
      75             : static bool isSMovRel(unsigned Opcode) {
      76             :   switch (Opcode) {
      77             :   case AMDGPU::S_MOVRELS_B32:
      78             :   case AMDGPU::S_MOVRELS_B64:
      79             :   case AMDGPU::S_MOVRELD_B32:
      80             :   case AMDGPU::S_MOVRELD_B64:
      81             :     return true;
      82             :   default:
      83             :     return false;
      84             :   }
      85             : }
      86             : 
      87             : static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
      88         264 :   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
      89             :                                                      AMDGPU::OpName::simm16);
      90         264 :   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
      91             : }
      92             : 
      93             : ScheduleHazardRecognizer::HazardType
      94      179548 : GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
      95      179548 :   MachineInstr *MI = SU->getInstr();
      96             : 
      97      179548 :   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
      98             :     return NoopHazard;
      99             : 
     100       21565 :   if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
     101             :     return NoopHazard;
     102             : 
     103      176749 :   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
     104             :     return NoopHazard;
     105             : 
     106      175000 :   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
     107             :     return NoopHazard;
     108             : 
     109      349972 :   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
     110             :     return NoopHazard;
     111             : 
     112      349938 :   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
     113             :     return NoopHazard;
     114             : 
     115      174965 :   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
     116             :     return NoopHazard;
     117             : 
     118      349930 :   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
     119             :     return NoopHazard;
     120             : 
     121      174965 :   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
     122             :     return NoopHazard;
     123             : 
     124      349930 :   if ((TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
     125         334 :       checkReadM0Hazards(MI) > 0)
     126             :     return NoopHazard;
     127             : 
     128      174965 :   if (checkAnyInstHazards(MI) > 0)
     129             :     return NoopHazard;
     130             : 
     131      174965 :   return NoHazard;
     132             : }
     133             : 
     134      174965 : unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
     135      174965 :   return PreEmitNoops(SU->getInstr());
     136             : }
     137             : 
     138      441989 : unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
     139      883978 :   int WaitStates = std::max(0, checkAnyInstHazards(MI));
     140             : 
     141      441989 :   if (SIInstrInfo::isSMRD(*MI))
     142      102296 :     return std::max(WaitStates, checkSMRDHazards(MI));
     143             : 
     144      390841 :   if (SIInstrInfo::isVALU(*MI)) {
     145      385566 :       WaitStates = std::max(WaitStates, checkVALUHazards(MI));
     146             : 
     147             :     if (SIInstrInfo::isVMEM(*MI))
     148           0 :       WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
     149             : 
     150      192783 :     if (SIInstrInfo::isDPP(*MI))
     151          58 :       WaitStates = std::max(WaitStates, checkDPPHazards(MI));
     152             : 
     153      578349 :     if (isDivFMas(MI->getOpcode()))
     154         470 :       WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
     155             : 
     156      578349 :     if (isRWLane(MI->getOpcode()))
     157          52 :       WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
     158             : 
     159      192783 :     if (TII.isVINTRP(*MI))
     160           0 :       WaitStates = std::max(WaitStates, checkReadM0Hazards(MI));
     161             : 
     162      192783 :     return WaitStates;
     163             :   }
     164             : 
     165      396116 :   if (isSGetReg(MI->getOpcode()))
     166          88 :     return std::max(WaitStates, checkGetRegHazards(MI));
     167             : 
     168      396028 :   if (isSSetReg(MI->getOpcode()))
     169         276 :     return std::max(WaitStates, checkSetRegHazards(MI));
     170             : 
     171      197876 :   if (isRFE(MI->getOpcode()))
     172          16 :     return std::max(WaitStates, checkRFEHazards(MI));
     173             : 
     174      197868 :   if (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))
     175        1674 :     return std::max(WaitStates, checkReadM0Hazards(MI));
     176             : 
     177      197031 :   return WaitStates;
     178             : }
     179             : 
     180        1570 : void GCNHazardRecognizer::EmitNoop() {
     181        3140 :   EmittedInstrs.push_front(nullptr);
     182        1570 : }
     183             : 
     184      879356 : void GCNHazardRecognizer::AdvanceCycle() {
     185             :   // When the scheduler detects a stall, it will call AdvanceCycle() without
     186             :   // emitting any instructions.
     187      879356 :   if (!CurrCycleInstr)
     188      437367 :     return;
     189             : 
     190      441989 :   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
     191             : 
     192             :   // Keep track of emitted instructions
     193      883978 :   EmittedInstrs.push_front(CurrCycleInstr);
     194             : 
     195             :   // Add a nullptr for each additional wait state after the first.  Make sure
     196             :   // not to add more than getMaxLookAhead() items to the list, since we
     197             :   // truncate the list to that size right after this loop.
     198      883978 :   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
     199      441989 :        i < e; ++i) {
     200           0 :     EmittedInstrs.push_front(nullptr);
     201             :   }
     202             : 
     203             :   // getMaxLookahead() is the largest number of wait states we will ever need
     204             :   // to insert, so there is no point in keeping track of more than that many
     205             :   // wait states.
     206      441989 :   EmittedInstrs.resize(getMaxLookAhead());
     207             : 
     208      441989 :   CurrCycleInstr = nullptr;
     209             : }
     210             : 
     211           0 : void GCNHazardRecognizer::RecedeCycle() {
     212           0 :   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
     213             : }
     214             : 
     215             : //===----------------------------------------------------------------------===//
     216             : // Helper Functions
     217             : //===----------------------------------------------------------------------===//
     218             : 
     219      358138 : int GCNHazardRecognizer::getWaitStatesSince(
     220             :     function_ref<bool(MachineInstr *)> IsHazard) {
     221      358138 :   int WaitStates = 0;
     222     2774941 :   for (MachineInstr *MI : EmittedInstrs) {
     223     1700527 :     if (MI) {
     224     1534844 :       if (IsHazard(MI))
     225             :         return WaitStates;
     226             : 
     227     3046844 :       unsigned Opcode = MI->getOpcode();
     228     1523422 :       if (Opcode == AMDGPU::DBG_VALUE || Opcode == AMDGPU::IMPLICIT_DEF ||
     229             :           Opcode == AMDGPU::INLINEASM)
     230        2322 :         continue;
     231             :     }
     232     1686783 :     ++WaitStates;
     233             :   }
     234             :   return std::numeric_limits<int>::max();
     235             : }
     236             : 
     237      160720 : int GCNHazardRecognizer::getWaitStatesSinceDef(
     238             :     unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) {
     239      321440 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     240             : 
     241      682643 :   auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
     242     1326627 :     return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
     243      804704 :   };
     244             : 
     245      321440 :   return getWaitStatesSince(IsHazardFn);
     246             : }
     247             : 
     248         198 : int GCNHazardRecognizer::getWaitStatesSinceSetReg(
     249             :     function_ref<bool(MachineInstr *)> IsHazard) {
     250             :   auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
     251        2038 :     return isSSetReg(MI->getOpcode()) && IsHazard(MI);
     252         198 :   };
     253             : 
     254         396 :   return getWaitStatesSince(IsHazardFn);
     255             : }
     256             : 
     257             : //===----------------------------------------------------------------------===//
     258             : // No-op Hazard Detection
     259             : //===----------------------------------------------------------------------===//
     260             : 
     261       82244 : static void addRegsToSet(iterator_range<MachineInstr::const_mop_iterator> Ops,
     262             :                          std::set<unsigned> &Set) {
     263      246714 :   for (const MachineOperand &Op : Ops) {
     264      164470 :     if (Op.isReg())
     265      164476 :       Set.insert(Op.getReg());
     266             :   }
     267       82244 : }
     268             : 
     269       75630 : int GCNHazardRecognizer::checkSMEMSoftClauseHazards(MachineInstr *SMEM) {
     270             :   // SMEM soft clause are only present on VI+
     271       75630 :   if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
     272             :     return 0;
     273             : 
     274             :   // A soft-clause is any group of consecutive SMEM instructions.  The
     275             :   // instructions in this group may return out of order and/or may be
     276             :   // replayed (i.e. the same instruction issued more than once).
     277             :   //
     278             :   // In order to handle these situations correctly we need to make sure
     279             :   // that when a clause has more than one instruction, no instruction in the
     280             :   // clause writes to a register that is read another instruction in the clause
     281             :   // (including itself). If we encounter this situaion, we need to break the
     282             :   // clause by inserting a non SMEM instruction.
     283             : 
     284       46499 :   std::set<unsigned> ClauseDefs;
     285       92998 :   std::set<unsigned> ClauseUses;
     286             : 
     287      193240 :   for (MachineInstr *MI : EmittedInstrs) {
     288             : 
     289             :     // When we hit a non-SMEM instruction then we have passed the start of the
     290             :     // clause and we can stop.
     291       90548 :     if (!MI || !SIInstrInfo::isSMRD(*MI))
     292             :       break;
     293             : 
     294       49136 :     addRegsToSet(MI->defs(), ClauseDefs);
     295       49136 :     addRegsToSet(MI->uses(), ClauseUses);
     296             :   }
     297             : 
     298       46499 :   if (ClauseDefs.empty())
     299             :     return 0;
     300             : 
     301             :   // FIXME: When we support stores, we need to make sure not to put loads and
     302             :   // stores in the same clause if they use the same address.  For now, just
     303             :   // start a new clause whenever we see a store.
     304       16554 :   if (SMEM->mayStore())
     305             :     return 1;
     306             : 
     307       33108 :   addRegsToSet(SMEM->defs(), ClauseDefs);
     308       33108 :   addRegsToSet(SMEM->uses(), ClauseUses);
     309             : 
     310       82770 :   std::vector<unsigned> Result(std::max(ClauseDefs.size(), ClauseUses.size()));
     311       16554 :   std::vector<unsigned>::iterator End;
     312             : 
     313       16554 :   End = std::set_intersection(ClauseDefs.begin(), ClauseDefs.end(),
     314       82770 :                               ClauseUses.begin(), ClauseUses.end(), Result.begin());
     315             : 
     316             :   // If the set of defs and uses intersect then we cannot add this instruction
     317             :   // to the clause, so we have a hazard.
     318       33108 :   if (End != Result.begin())
     319             :     return 1;
     320             : 
     321       13800 :   return 0;
     322             : }
     323             : 
     324       75630 : int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
     325       75630 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
     326             :   int WaitStatesNeeded = 0;
     327             : 
     328       75630 :   WaitStatesNeeded = checkSMEMSoftClauseHazards(SMRD);
     329             : 
     330             :   // This SMRD hazard only affects SI.
     331       75630 :   if (ST.getGeneration() != SISubtarget::SOUTHERN_ISLANDS)
     332             :     return WaitStatesNeeded;
     333             : 
     334             :   // A read of an SGPR by SMRD instruction requires 4 wait states when the
     335             :   // SGPR was written by a VALU instruction.
     336       19553 :   int SmrdSgprWaitStates = 4;
     337       48531 :   auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
     338             : 
     339       78180 :   for (const MachineOperand &Use : SMRD->uses()) {
     340       58627 :     if (!Use.isReg())
     341       39054 :       continue;
     342             :     int WaitStatesNeededForUse =
     343       39146 :         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
     344       19573 :     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
     345             :   }
     346       19553 :   return WaitStatesNeeded;
     347             : }
     348             : 
     349       21565 : int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
     350       43130 :   const SIInstrInfo *TII = ST.getInstrInfo();
     351             : 
     352       21565 :   if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
     353             :     return 0;
     354             : 
     355       12580 :   const SIRegisterInfo &TRI = TII->getRegisterInfo();
     356             : 
     357             :   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
     358             :   // SGPR was written by a VALU Instruction.
     359       12580 :   int VmemSgprWaitStates = 5;
     360       12580 :   int WaitStatesNeeded = 0;
     361      152361 :   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
     362             : 
     363      110662 :   for (const MachineOperand &Use : VMEM->uses()) {
     364       98082 :     if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
     365       68734 :       continue;
     366             : 
     367             :     int WaitStatesNeededForUse =
     368       58696 :         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
     369       29348 :     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
     370             :   }
     371       12580 :   return WaitStatesNeeded;
     372             : }
     373             : 
     374          50 : int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
     375         100 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     376         100 :   const SIInstrInfo *TII = ST.getInstrInfo();
     377             : 
     378             :   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
     379          50 :   int DppVgprWaitStates = 2;
     380          50 :   int DppExecWaitStates = 5;
     381          50 :   int WaitStatesNeeded = 0;
     382         243 :   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
     383             : 
     384         400 :   for (const MachineOperand &Use : DPP->uses()) {
     385         350 :     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
     386         250 :       continue;
     387             :     int WaitStatesNeededForUse =
     388         100 :         DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
     389         100 :     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
     390             :   }
     391             : 
     392          50 :   WaitStatesNeeded = std::max(
     393             :       WaitStatesNeeded,
     394         200 :       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn));
     395             : 
     396          50 :   return WaitStatesNeeded;
     397             : }
     398             : 
     399         348 : int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
     400         696 :   const SIInstrInfo *TII = ST.getInstrInfo();
     401             : 
     402             :   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
     403             :   // instruction.
     404         348 :   const int DivFMasWaitStates = 4;
     405        1802 :   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
     406         696 :   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn);
     407             : 
     408         348 :   return DivFMasWaitStates - WaitStatesNeeded;
     409             : }
     410             : 
     411          56 : int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
     412         112 :   const SIInstrInfo *TII = ST.getInstrInfo();
     413          56 :   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
     414             : 
     415          56 :   const int GetRegWaitStates = 2;
     416          16 :   auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
     417          16 :     return GetRegHWReg == getHWReg(TII, *MI);
     418          72 :   };
     419         112 :   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
     420             : 
     421          56 :   return GetRegWaitStates - WaitStatesNeeded;
     422             : }
     423             : 
     424         138 : int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
     425         276 :   const SIInstrInfo *TII = ST.getInstrInfo();
     426         138 :   unsigned HWReg = getHWReg(TII, *SetRegInstr);
     427             : 
     428             :   const int SetRegWaitStates =
     429         138 :       ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ? 1 : 2;
     430          48 :   auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
     431          48 :     return HWReg == getHWReg(TII, *MI);
     432         186 :   };
     433         276 :   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
     434         138 :   return SetRegWaitStates - WaitStatesNeeded;
     435             : }
     436             : 
     437      890204 : int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
     438      890204 :   if (!MI.mayStore())
     439             :     return -1;
     440             : 
     441       69342 :   const SIInstrInfo *TII = ST.getInstrInfo();
     442       69342 :   unsigned Opcode = MI.getOpcode();
     443       34671 :   const MCInstrDesc &Desc = MI.getDesc();
     444             : 
     445       34671 :   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
     446       34671 :   int VDataRCID = -1;
     447       34671 :   if (VDataIdx != -1)
     448       29755 :     VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
     449             : 
     450       50338 :   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
     451             :     // There is no hazard if the instruction does not use vector regs
     452             :     // (like wbinvl1)
     453       19004 :     if (VDataIdx == -1)
     454             :       return -1;
     455             :     // For MUBUF/MTBUF instructions this hazard only exists if the
     456             :     // instruction is not using a register in the soffset field.
     457       18998 :     const MachineOperand *SOffset =
     458             :         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
     459             :     // If we have no soffset operand, then assume this field has been
     460             :     // hardcoded to zero.
     461       18998 :     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
     462        9938 :         (!SOffset || !SOffset->isReg()))
     463             :       return VDataIdx;
     464             :   }
     465             : 
     466             :   // MIMG instructions create a hazard if they don't use a 256-bit T# and
     467             :   // the store size is greater than 8 bytes and they have more than two bits
     468             :   // of their dmask set.
     469             :   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
     470       24805 :   if (TII->isMIMG(MI)) {
     471             :     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
     472             :     assert(SRsrcIdx != -1 &&
     473             :            AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
     474             :     (void)SRsrcIdx;
     475             :   }
     476             : 
     477       24805 :   if (TII->isFLAT(MI)) {
     478       10710 :     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
     479       10710 :     if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
     480             :       return DataIdx;
     481             :   }
     482             : 
     483             :   return -1;
     484             : }
     485             : 
     486      276005 : int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
     487             :   // This checks for the hazard where VMEM instructions that store more than
     488             :   // 8 bytes can have there store data over written by the next instruction.
     489      552010 :   if (!ST.has12DWordStoreHazard())
     490             :     return 0;
     491             : 
     492      411518 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     493      205759 :   const MachineRegisterInfo &MRI = VALU->getParent()->getParent()->getRegInfo();
     494             : 
     495      205759 :   const int VALUWaitStates = 1;
     496      205759 :   int WaitStatesNeeded = 0;
     497             : 
     498      411592 :   for (const MachineOperand &Def : VALU->defs()) {
     499      205833 :     if (!TRI->isVGPR(MRI, Def.getReg()))
     500        8613 :       continue;
     501      197220 :     unsigned Reg = Def.getReg();
     502      925812 :     auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
     503      890204 :       int DataIdx = createsVALUHazard(*MI);
     504      908008 :       return DataIdx >= 0 &&
     505      943616 :              TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
     506      197220 :     };
     507             :     int WaitStatesNeededForDef =
     508      394440 :         VALUWaitStates - getWaitStatesSince(IsHazardFn);
     509      197220 :     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
     510             :   }
     511      205759 :   return WaitStatesNeeded;
     512             : }
     513             : 
     514          35 : int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
     515          70 :   const SIInstrInfo *TII = ST.getInstrInfo();
     516          70 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     517             :   const MachineRegisterInfo &MRI =
     518          35 :       RWLane->getParent()->getParent()->getRegInfo();
     519             : 
     520          35 :   const MachineOperand *LaneSelectOp =
     521             :       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
     522             : 
     523          35 :   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
     524             :     return 0;
     525             : 
     526          32 :   unsigned LaneSelectReg = LaneSelectOp->getReg();
     527             :   auto IsHazardFn = [TII] (MachineInstr *MI) {
     528             :     return TII->isVALU(*MI);
     529         104 :   };
     530             : 
     531          32 :   const int RWLaneWaitStates = 4;
     532          64 :   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
     533          32 :   return RWLaneWaitStates - WaitStatesSince;
     534             : }
     535             : 
     536           8 : int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
     537           8 :   if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
     538             :     return 0;
     539             : 
     540           8 :   const SIInstrInfo *TII = ST.getInstrInfo();
     541             : 
     542           4 :   const int RFEWaitStates = 1;
     543             : 
     544           6 :   auto IsHazardFn = [TII] (MachineInstr *MI) {
     545           6 :     return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
     546          10 :   };
     547           8 :   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
     548           4 :   return RFEWaitStates - WaitStatesNeeded;
     549             : }
     550             : 
     551      616954 : int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
     552      616954 :   if (MI->isDebugValue())
     553             :     return 0;
     554             : 
     555     1233886 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     556     1233886 :   if (!ST.hasSMovFedHazard())
     557             :     return 0;
     558             : 
     559             :   // Check for any instruction reading an SGPR after a write from
     560             :   // s_mov_fed_b32.
     561       62596 :   int MovFedWaitStates = 1;
     562       62596 :   int WaitStatesNeeded = 0;
     563             : 
     564      304359 :   for (const MachineOperand &Use : MI->uses()) {
     565      241763 :     if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
     566      130504 :       continue;
     567             :     auto IsHazardFn = [] (MachineInstr *MI) {
     568      473332 :       return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
     569      473332 :     };
     570             :     int WaitStatesNeededForUse =
     571      222518 :         MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn);
     572      111259 :     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
     573             :   }
     574             : 
     575       62596 :   return WaitStatesNeeded;
     576             : }
     577             : 
     578        1171 : int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
     579        2342 :   if (!ST.hasReadM0Hazard())
     580             :     return 0;
     581             : 
     582          20 :   const SIInstrInfo *TII = ST.getInstrInfo();
     583          10 :   int SMovRelWaitStates = 1;
     584             :   auto IsHazardFn = [TII] (MachineInstr *MI) {
     585             :     return TII->isSALU(*MI);
     586          22 :   };
     587          20 :   return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn);
     588             : }

Generated by: LCOV version 1.13