#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/TargetParser/TargetParser.h"

Enumerations
enum	HazardFnResult { HazardFound , HazardExpired , NoHazardFound }

Functions
static bool	shouldRunLdsBranchVmemWARHazardFixup (const MachineFunction &MF, const GCNSubtarget &ST)
static bool	isDivFMas (unsigned Opcode)
static bool	isSGetReg (unsigned Opcode)
static bool	isSSetReg (unsigned Opcode)
static bool	isRWLane (unsigned Opcode)
static bool	isRFE (unsigned Opcode)
static bool	isSMovRel (unsigned Opcode)
static bool	isSendMsgTraceDataOrGDS (const SIInstrInfo &TII, const MachineInstr &MI)
static bool	isPermlane (const MachineInstr &MI)
static bool	isLdsDma (const MachineInstr &MI)
static unsigned	getHWReg (const SIInstrInfo *TII, const MachineInstr &RegInstr)
static void	insertNoopsInBundle (MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
template<typename StateT>
static bool	hasHazard (StateT InitialState, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *InitialMBB, MachineBasicBlock::const_reverse_instr_iterator InitialI)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock > &Visited, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, GCNHazardRecognizer::IsExpiredFn IsExpired, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static void	addRegUnits (const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static void	addRegsToSet (const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static bool	breaksSMEMSoftClause (MachineInstr *MI)
static bool	breaksVMEMSoftClause (MachineInstr *MI)
static const MachineOperand *	getDstSelForwardingOperand (const MachineInstr &MI, const GCNSubtarget &ST)
	Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.
static bool	consumesDstSelForwardingOperand (const MachineInstr VALU, const MachineOperand Dst, const SIRegisterInfo *TRI)
	Checks whether the provided `MI` "consumes" the operand with a Dest sel fowarding issue `Dst` .
static bool	isVCmpXWritesExec (const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool	isStoreCountWaitZero (const MachineInstr &I)
static bool	isCoexecutableVALUInst (const MachineInstr &MI)
static bool	IsWMMAHazardInstInCategory (const MachineInstr &MI, const SIInstrInfo *TII, unsigned Latency, unsigned Category)
static int	GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWriteVgprVALUWawWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
static void	updateGetPCBundle (MachineInstr *NewMI)
static bool	ensureEntrySetPrio (MachineFunction *MF, int Priority, const SIInstrInfo &TII)

Variables
static cl::opt< unsigned, false, MFMAPaddingRatioParser >	MFMAPaddingRatio ("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static cl::opt< unsigned >	NopPadding ("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction"))

Enumeration Type Documentation

◆ HazardFnResult

enum HazardFnResult

Enumerator
HazardFound
HazardExpired
NoHazardFound

Definition at line 444 of file GCNHazardRecognizer.cpp.

Function Documentation

◆ addRegsToSet()

void addRegsToSet	(	const SIRegisterInfo &	TRI,
		iterator_range< MachineInstr::const_mop_iterator >	Ops,
		BitVector &	DefSet,
		BitVector &	UseSet )

static

Definition at line 665 of file GCNHazardRecognizer.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, addRegUnits(), and TRI.

◆ addRegUnits()

void addRegUnits	(	const SIRegisterInfo &	TRI,
		BitVector &	BV,
		MCRegister	Reg )

static

Definition at line 659 of file GCNHazardRecognizer.cpp.

References Reg, llvm::BitVector::set(), and TRI.

Referenced by addRegsToSet().

◆ breaksSMEMSoftClause()

bool breaksSMEMSoftClause ( MachineInstr * MI )

static

Definition at line 678 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSMRD(), and MI.

◆ breaksVMEMSoftClause()

bool breaksVMEMSoftClause ( MachineInstr * MI )

static

Definition at line 682 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isVMEM(), and MI.

◆ consumesDstSelForwardingOperand()

bool consumesDstSelForwardingOperand	(	const MachineInstr *	VALU,
		const MachineOperand *	Dst,
		const SIRegisterInfo *	TRI )

static

Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .

We may "consume" the Dst via a standard explicit RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)

Definition at line 1008 of file GCNHazardRecognizer.cpp.

References TRI.

◆ ensureEntrySetPrio()

bool ensureEntrySetPrio	(	MachineFunction *	MF,
		int	Priority,
		const SIInstrInfo &	TII )

static

Definition at line 3524 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineBasicBlock::end(), llvm::MachineFunction::front(), and TII.

◆ getDstSelForwardingOperand()

const MachineOperand * getDstSelForwardingOperand	(	const MachineInstr &	MI,
		const GCNSubtarget &	ST )

static

Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.

This occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with dst_sel that is not aligned to the register. This function analayzes the MI and

Returns: an operand with dst forwarding issue, or nullptr if none exists.

Definition at line 962 of file GCNHazardRecognizer.cpp.

References llvm::SISrcMods::DST_OP_SEL, llvm::AMDGPU::SDWA::DWORD, llvm::AMDGPU::FP4, llvm::AMDGPU::FP8, llvm::AMDGPU::getFPDstSelType(), llvm::AMDGPU::hasNamedOperand(), llvm::SIInstrInfo::isSDWA(), llvm::SIInstrInfo::isVALU(), MI, llvm::SISrcMods::OP_SEL_0, and TII.

◆ getHWReg()

unsigned getHWReg	(	const SIInstrInfo *	TII,
		const MachineInstr &	RegInstr )

static

Definition at line 168 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::EncodingFields< HwregId, HwregOffset, HwregSize >::decode(), llvm::MachineOperand::getImm(), and TII.

◆ getWaitStatesSince() [1/2]

int getWaitStatesSince	(	GCNHazardRecognizer::IsHazardFn	IsHazard,
		const MachineBasicBlock *	MBB,
		MachineBasicBlock::const_reverse_instr_iterator	I,
		int	WaitStates,
		GCNHazardRecognizer::IsExpiredFn	IsExpired,
		DenseSet< const MachineBasicBlock * > &	Visited,
		GCNHazardRecognizer::GetNumWaitStatesFn	GetNumWaitStates = SIInstrInfo::getNumWaitStates )

static

Definition at line 553 of file GCNHazardRecognizer.cpp.

References E(), llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and MBB.

Referenced by getWaitStatesSince(), and getWaitStatesSince().

◆ getWaitStatesSince() [2/2]

int getWaitStatesSince	(	GCNHazardRecognizer::IsHazardFn	IsHazard,
		const MachineInstr *	MI,
		GCNHazardRecognizer::IsExpiredFn	IsExpired,
		GCNHazardRecognizer::GetNumWaitStatesFn	GetNumWaitStates = SIInstrInfo::getNumWaitStates )

static

Definition at line 592 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), and MI.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int NumPasses )

static

Definition at line 2571 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates ( int NumPasses )

static

Definition at line 2580 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates ( int NumPasses )

static

Definition at line 2918 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates ( int NumPasses )

static

Definition at line 2890 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2560 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2588 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2549 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2908 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2898 of file GCNHazardRecognizer.cpp.

◆ hasHazard()

template<typename StateT>

bool hasHazard	(	StateT	InitialState,
		function_ref< HazardFnResult(StateT &, const MachineInstr &)>	IsHazard,
		function_ref< void(StateT &, const MachineInstr &)>	UpdateState,
		const MachineBasicBlock *	InitialMBB,
		MachineBasicBlock::const_reverse_instr_iterator	InitialI )

static

Definition at line 449 of file GCNHazardRecognizer.cpp.

References E(), llvm::SmallVectorImpl< T >::emplace_back(), HazardExpired, HazardFound, I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert_as(), isEqual(), LHS, MBB, RHS, llvm::SetVector< T, Vector, Set, N >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

◆ insertNoopsInBundle()

void insertNoopsInBundle	(	MachineInstr *	MI,
		const SIInstrInfo &	TII,
		unsigned	Quantity )

static

Definition at line 255 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), MI, and TII.

◆ isCoexecutableVALUInst()

bool isCoexecutableVALUInst ( const MachineInstr & MI )

static

Definition at line 2032 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isTRANS(), llvm::SIInstrInfo::isVALU(), llvm::SIInstrInfo::isWMMA(), and MI.

◆ isDivFMas()

bool isDivFMas ( unsigned Opcode )

static

Definition at line 80 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isLdsDma()

bool isLdsDma ( const MachineInstr & MI )

static

Definition at line 163 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isMUBUF(), llvm::SIInstrInfo::isVALU(), and MI.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isPermlane()

bool isPermlane ( const MachineInstr & MI )

static

Definition at line 145 of file GCNHazardRecognizer.cpp.

References MI.

Referenced by llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRFE()

bool isRFE ( unsigned Opcode )

static

Definition at line 103 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRWLane()

bool isRWLane ( unsigned Opcode )

static

Definition at line 99 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSendMsgTraceDataOrGDS()

bool isSendMsgTraceDataOrGDS	(	const SIInstrInfo &	TII,
		const MachineInstr &	MI )

static

Definition at line 119 of file GCNHazardRecognizer.cpp.

References MI, and TII.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSGetReg()

bool isSGetReg ( unsigned Opcode )

static

Definition at line 84 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSMovRel()

bool isSMovRel ( unsigned Opcode )

static

Definition at line 107 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSSetReg()

bool isSSetReg ( unsigned Opcode )

static

Definition at line 88 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isStoreCountWaitZero()

bool isStoreCountWaitZero ( const MachineInstr & I )

static

Definition at line 1544 of file GCNHazardRecognizer.cpp.

References I.

◆ isVCmpXWritesExec()

bool isVCmpXWritesExec	(	const SIInstrInfo &	TII,
		const SIRegisterInfo &	TRI,
		const MachineInstr &	MI )

static

Definition at line 1304 of file GCNHazardRecognizer.cpp.

References MI, TII, and TRI.

◆ IsWMMAHazardInstInCategory()

bool IsWMMAHazardInstInCategory	(	const MachineInstr &	MI,
		const SIInstrInfo *	TII,
		unsigned	Latency,
		unsigned	Category )

static

Definition at line 2037 of file GCNHazardRecognizer.cpp.

References assert(), llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isWMMA(), llvm::Latency, MI, and TII.

◆ shouldRunLdsBranchVmemWARHazardFixup()

bool shouldRunLdsBranchVmemWARHazardFixup	(	const MachineFunction &	MF,
		const GCNSubtarget &	ST )

static

Definition at line 1524 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isDS(), llvm::SIInstrInfo::isLDSDMA(), llvm::SIInstrInfo::isVMEM(), MBB, and MI.

Referenced by llvm::GCNHazardRecognizer::GCNHazardRecognizer().

◆ updateGetPCBundle()

void updateGetPCBundle ( MachineInstr * NewMI )

static

Definition at line 3248 of file GCNHazardRecognizer.cpp.

References assert(), llvm::MachineBasicBlock::end(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getParent(), I, and llvm::MachineInstr::isBundled().

Variable Documentation

◆ MFMAPaddingRatio

cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))	(	"amdgpu-mfma-padding-ratio"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")	)

static

◆ NopPadding

cl::opt< unsigned > NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction"))	(	"amdgpu-snop-padding"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Insert a s_nop x before every instruction")	)

static

Referenced by llvm::GCNHazardRecognizer::PreEmitNoops().

Enumerations

Functions

Variables

Enumeration Type Documentation

◆ HazardFnResult

Function Documentation

◆ addRegsToSet()

◆ addRegUnits()

◆ breaksSMEMSoftClause()

◆ breaksVMEMSoftClause()

◆ consumesDstSelForwardingOperand()

◆ ensureEntrySetPrio()

◆ getDstSelForwardingOperand()

◆ getHWReg()

◆ getWaitStatesSince() [1/2]

◆ getWaitStatesSince() [2/2]

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

◆ hasHazard()

◆ insertNoopsInBundle()

◆ isCoexecutableVALUInst()

◆ isDivFMas()

◆ isLdsDma()

◆ isPermlane()

◆ isRFE()

◆ isRWLane()

◆ isSendMsgTraceDataOrGDS()

◆ isSGetReg()

◆ isSMovRel()

◆ isSSetReg()

◆ isStoreCountWaitZero()

◆ isVCmpXWritesExec()

◆ IsWMMAHazardInstInCategory()

◆ shouldRunLdsBranchVmemWARHazardFixup()

◆ updateGetPCBundle()

Variable Documentation

◆ MFMAPaddingRatio

◆ NopPadding