LLVM 20.0.0git
Typedefs | Functions | Variables
GCNHazardRecognizer.cpp File Reference
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/TargetParser/TargetParser.h"

Go to the source code of this file.

Typedefs

using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }
 
using IsExpiredFn = function_ref< bool(const MachineInstr &, int WaitStates)>
 
using GetNumWaitStatesFn = function_ref< unsigned int(const MachineInstr &)>
 

Functions

static bool shouldRunLdsBranchVmemWARHazardFixup (const MachineFunction &MF, const GCNSubtarget &ST)
 
static bool isDivFMas (unsigned Opcode)
 
static bool isSGetReg (unsigned Opcode)
 
static bool isSSetReg (unsigned Opcode)
 
static bool isRWLane (unsigned Opcode)
 
static bool isRFE (unsigned Opcode)
 
static bool isSMovRel (unsigned Opcode)
 
static bool isDGEMM (unsigned Opcode)
 
static bool isXDL (const GCNSubtarget &ST, const MachineInstr &MI)
 
static bool isSendMsgTraceDataOrGDS (const SIInstrInfo &TII, const MachineInstr &MI)
 
static bool isPermlane (const MachineInstr &MI)
 
static bool isLdsDma (const MachineInstr &MI)
 
static unsigned getHWReg (const SIInstrInfo *TII, const MachineInstr &RegInstr)
 
static void insertNoopsInBundle (MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
 
template<typename StateT >
static bool hasHazard (StateT State, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, DenseSet< const MachineBasicBlock * > &Visited)
 
static int getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
 
static int getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, IsExpiredFn IsExpired)
 
static void addRegUnits (const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
 
static void addRegsToSet (const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
 
static bool breaksSMEMSoftClause (MachineInstr *MI)
 
static bool breaksVMEMSoftClause (MachineInstr *MI)
 
static bool isStoreCountWaitZero (const MachineInstr &I)
 
static int GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
 
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
 
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
 
static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
 
static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
 
static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
 
static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
 
static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
 
static bool ensureEntrySetPrio (MachineFunction *MF, int Priority, const SIInstrInfo &TII)
 

Variables

static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio ("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
 

Typedef Documentation

◆ GetNumWaitStatesFn

using GetNumWaitStatesFn = function_ref<unsigned int(const MachineInstr &)>

Definition at line 453 of file GCNHazardRecognizer.cpp.

◆ HazardFnResult

using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }

Definition at line 450 of file GCNHazardRecognizer.cpp.

◆ IsExpiredFn

using IsExpiredFn = function_ref<bool(const MachineInstr &, int WaitStates)>

Definition at line 452 of file GCNHazardRecognizer.cpp.

Function Documentation

◆ addRegsToSet()

static void addRegsToSet ( const SIRegisterInfo &  TRI,
iterator_range< MachineInstr::const_mop_iterator >  Ops,
BitVector &  DefSet,
BitVector &  UseSet 
)
static

Definition at line 600 of file GCNHazardRecognizer.cpp.

References addRegUnits(), and TRI.

◆ addRegUnits()

static void addRegUnits ( const SIRegisterInfo &  TRI,
BitVector &  BV,
MCRegister  Reg 
)
static

Definition at line 594 of file GCNHazardRecognizer.cpp.

References llvm::BitVector::set(), and TRI.

Referenced by addRegsToSet().

◆ breaksSMEMSoftClause()

static bool breaksSMEMSoftClause ( MachineInstr *  MI)
static

Definition at line 613 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSMRD(), and MI.

◆ breaksVMEMSoftClause()

static bool breaksVMEMSoftClause ( MachineInstr *  MI)
static

Definition at line 617 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isVMEM(), and MI.

◆ ensureEntrySetPrio()

static bool ensureEntrySetPrio ( MachineFunction *  MF,
int  Priority,
const SIInstrInfo &  TII 
)
static

Definition at line 2901 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineBasicBlock::end(), llvm::MachineFunction::front(), and TII.

◆ getHWReg()

static unsigned getHWReg ( const SIInstrInfo *  TII,
const MachineInstr &  RegInstr 
)
static

Definition at line 178 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::EncodingFields< Fields >::decode(), llvm::MachineOperand::getImm(), and TII.

◆ getWaitStatesSince() [1/2]

static int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn  IsHazard,
const MachineBasicBlock *  MBB,
MachineBasicBlock::const_reverse_instr_iterator  I,
int  WaitStates,
IsExpiredFn  IsExpired,
DenseSet< const MachineBasicBlock * > &  Visited,
GetNumWaitStatesFn  GetNumWaitStates = SIInstrInfo::getNumWaitStates 
)
static

Definition at line 500 of file GCNHazardRecognizer.cpp.

References getWaitStatesSince(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::MachineBasicBlock::instr_rend(), MBB, and llvm::MachineBasicBlock::predecessors().

Referenced by getWaitStatesSince().

◆ getWaitStatesSince() [2/2]

static int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn  IsHazard,
const MachineInstr *  MI,
IsExpiredFn  IsExpired 
)
static

Definition at line 536 of file GCNHazardRecognizer.cpp.

References getWaitStatesSince(), and MI.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int  NumPasses)
static

Definition at line 2152 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates ( int  NumPasses)
static

Definition at line 2161 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates ( int  NumPasses)
static

Definition at line 2440 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates ( int  NumPasses)
static

Definition at line 2416 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

static int GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int  NumPasses)
static

Definition at line 2143 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates ( int  NumPasses)
static

Definition at line 2169 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates ( int  NumPasses)
static

Definition at line 2432 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates ( int  NumPasses)
static

Definition at line 2424 of file GCNHazardRecognizer.cpp.

◆ hasHazard()

template<typename StateT >
static bool hasHazard ( StateT  State,
function_ref< HazardFnResult(StateT &, const MachineInstr &)>  IsHazard,
function_ref< void(StateT &, const MachineInstr &)>  UpdateState,
const MachineBasicBlock *  MBB,
MachineBasicBlock::const_reverse_instr_iterator  I,
DenseSet< const MachineBasicBlock * > &  Visited 
)
static

Definition at line 458 of file GCNHazardRecognizer.cpp.

References hasHazard(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::MachineBasicBlock::instr_rend(), MBB, and llvm::MachineBasicBlock::predecessors().

Referenced by hasHazard().

◆ insertNoopsInBundle()

static void insertNoopsInBundle ( MachineInstr *  MI,
const SIInstrInfo &  TII,
unsigned  Quantity 
)
static

Definition at line 263 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), MI, and TII.

◆ isDGEMM()

static bool isDGEMM ( unsigned  Opcode)
static

Definition at line 119 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getMAIIsDGEMM().

Referenced by isXDL().

◆ isDivFMas()

static bool isDivFMas ( unsigned  Opcode)
static

Definition at line 80 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isLdsDma()

static bool isLdsDma ( const MachineInstr &  MI)
static

Definition at line 173 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isMUBUF(), llvm::SIInstrInfo::isVALU(), and MI.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isPermlane()

static bool isPermlane ( const MachineInstr &  MI)
static

Definition at line 164 of file GCNHazardRecognizer.cpp.

References MI.

◆ isRFE()

static bool isRFE ( unsigned  Opcode)
static

Definition at line 103 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRWLane()

static bool isRWLane ( unsigned  Opcode)
static

Definition at line 99 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSendMsgTraceDataOrGDS()

static bool isSendMsgTraceDataOrGDS ( const SIInstrInfo &  TII,
const MachineInstr &  MI 
)
static

Definition at line 138 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getNamedOperandIdx(), MI, and TII.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSGetReg()

static bool isSGetReg ( unsigned  Opcode)
static

Definition at line 84 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSMovRel()

static bool isSMovRel ( unsigned  Opcode)
static

Definition at line 107 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSSetReg()

static bool isSSetReg ( unsigned  Opcode)
static

Definition at line 88 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isStoreCountWaitZero()

static bool isStoreCountWaitZero ( const MachineInstr &  I)
static

Definition at line 1345 of file GCNHazardRecognizer.cpp.

References I.

◆ isXDL()

static bool isXDL ( const GCNSubtarget &  ST,
const MachineInstr &  MI 
)
static

Definition at line 123 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getMAIIsGFX940XDL(), isDGEMM(), llvm::SIInstrInfo::isMAI(), and MI.

◆ shouldRunLdsBranchVmemWARHazardFixup()

static bool shouldRunLdsBranchVmemWARHazardFixup ( const MachineFunction &  MF,
const GCNSubtarget &  ST 
)
static

Definition at line 1324 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isDS(), llvm::SIInstrInfo::isSegmentSpecificFLAT(), llvm::SIInstrInfo::isVMEM(), MBB, and MI.

Referenced by llvm::GCNHazardRecognizer::GCNHazardRecognizer().

Variable Documentation

◆ MFMAPaddingRatio

cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")) ( "amdgpu-mfma-padding-ratio"  ,
cl::init(0)  ,
cl::Hidden  ,
cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")   
)
static
Generated on Fri Nov 22 2024 22:33:38 for LLVM by doxygen 1.9.6