#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/TargetParser.h"

Macros
#define	DEBUG_TYPE "gcn-hazard-recognizer"

Enumerations
enum	HazardFnResult { HazardFound , HazardExpired , NoHazardFound }

Functions
	STATISTIC (NumWMMANopsHoisted, "Number of WMMA hazard V_NOPs hoisted from loops")
	STATISTIC (NumWMMAHoistingBailed, "Number of WMMA hazards where V_NOP hoisting was not possible")
static bool	shouldRunLdsBranchVmemWARHazardFixup (const MachineFunction &MF, const GCNSubtarget &ST)
static bool	isDivFMas (unsigned Opcode)
static bool	isSGetReg (unsigned Opcode)
static bool	isSSetReg (unsigned Opcode)
static bool	isRWLane (unsigned Opcode)
static bool	isRFE (unsigned Opcode)
static bool	isSMovRel (unsigned Opcode)
static bool	isSendMsgTraceDataOrGDS (const SIInstrInfo &TII, const MachineInstr &MI)
static bool	isPermlane (const MachineInstr &MI)
static bool	isLdsDma (const MachineInstr &MI)
static unsigned	getHWReg (const SIInstrInfo *TII, const MachineInstr &RegInstr)
static void	insertNoopsInBundle (MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
template<typename StateT>
static bool	hasHazard (StateT InitialState, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *InitialMBB, MachineBasicBlock::const_reverse_instr_iterator InitialI)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock > &Visited, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, GCNHazardRecognizer::IsExpiredFn IsExpired, GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static void	addRegUnits (const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static void	addRegsToSet (const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static bool	breaksSMEMSoftClause (MachineInstr *MI)
static bool	breaksVMEMSoftClause (MachineInstr *MI)
static const MachineOperand *	getDstSelForwardingOperand (const MachineInstr &MI, const GCNSubtarget &ST)
	Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.
static bool	consumesDstSelForwardingOperand (const MachineInstr VALU, const MachineOperand Dst, const SIRegisterInfo *TRI)
	Checks whether the provided `MI` "consumes" the operand with a Dest sel fowarding issue `Dst` .
static bool	isVCmpXWritesExec (const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool	isStoreCountWaitZero (const MachineInstr &I)
static bool	isCoexecutableVALUInst (const MachineInstr &MI)
static bool	IsWMMAHazardInstInCategory (const MachineInstr &MI, const SIInstrInfo *TII, unsigned Latency, unsigned Category)
static int	GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWriteVgprVALUWawWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
static void	updateGetPCBundle (MachineInstr *NewMI)
static bool	ensureEntrySetPrio (MachineFunction *MF, int Priority, const SIInstrInfo &TII)

Variables
static cl::opt< unsigned, false, MFMAPaddingRatioParser >	MFMAPaddingRatio ("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static cl::opt< unsigned >	NopPadding ("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction"))
static cl::opt< bool >	EnableWMMAVnopHoisting ("amdgpu-wmma-vnop-hoisting", cl::init(true), cl::Hidden, cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"))

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "gcn-hazard-recognizer"

Definition at line 27 of file GCNHazardRecognizer.cpp.

Enumeration Type Documentation

◆ HazardFnResult

enum HazardFnResult

Enumerator
HazardFound
HazardExpired
NoHazardFound

Definition at line 459 of file GCNHazardRecognizer.cpp.

Function Documentation

◆ addRegsToSet()

void addRegsToSet	(	const SIRegisterInfo &	TRI,
		iterator_range< MachineInstr::const_mop_iterator >	Ops,
		BitVector &	DefSet,
		BitVector &	UseSet )

static

Definition at line 680 of file GCNHazardRecognizer.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, addRegUnits(), and TRI.

◆ addRegUnits()

void addRegUnits	(	const SIRegisterInfo &	TRI,
		BitVector &	BV,
		MCRegister	Reg )

static

Definition at line 674 of file GCNHazardRecognizer.cpp.

References Reg, llvm::BitVector::set(), and TRI.

Referenced by addRegsToSet().

◆ breaksSMEMSoftClause()

bool breaksSMEMSoftClause ( MachineInstr * MI )

static

Definition at line 693 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSMRD(), and MI.

◆ breaksVMEMSoftClause()

bool breaksVMEMSoftClause ( MachineInstr * MI )

static

Definition at line 697 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isVMEM(), and MI.

◆ consumesDstSelForwardingOperand()

bool consumesDstSelForwardingOperand	(	const MachineInstr *	VALU,
		const MachineOperand *	Dst,
		const SIRegisterInfo *	TRI )

static

Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .

We may "consume" the Dst via a standard explicit RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)

Definition at line 1023 of file GCNHazardRecognizer.cpp.

References TRI.

◆ ensureEntrySetPrio()

bool ensureEntrySetPrio	(	MachineFunction *	MF,
		int	Priority,
		const SIInstrInfo &	TII )

static

Definition at line 3637 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineBasicBlock::end(), llvm::MachineFunction::front(), and TII.

◆ getDstSelForwardingOperand()

const MachineOperand * getDstSelForwardingOperand	(	const MachineInstr &	MI,
		const GCNSubtarget &	ST )

static

Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.

This occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with dst_sel that is not aligned to the register. This function analayzes the MI and

Returns: an operand with dst forwarding issue, or nullptr if none exists.

Definition at line 977 of file GCNHazardRecognizer.cpp.

References llvm::SISrcMods::DST_OP_SEL, llvm::AMDGPU::SDWA::DWORD, llvm::AMDGPU::FP4, llvm::AMDGPU::FP8, llvm::AMDGPU::getFPDstSelType(), llvm::AMDGPU::hasNamedOperand(), llvm::SIInstrInfo::isSDWA(), llvm::SIInstrInfo::isVALU(), MI, llvm::SISrcMods::OP_SEL_0, and TII.

◆ getHWReg()

unsigned getHWReg	(	const SIInstrInfo *	TII,
		const MachineInstr &	RegInstr )

static

Definition at line 183 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::EncodingFields< HwregId, HwregOffset, HwregSize >::decode(), llvm::MachineOperand::getImm(), and TII.

◆ getWaitStatesSince() [1/2]

int getWaitStatesSince	(	GCNHazardRecognizer::IsHazardFn	IsHazard,
		const MachineBasicBlock *	MBB,
		MachineBasicBlock::const_reverse_instr_iterator	I,
		int	WaitStates,
		GCNHazardRecognizer::IsExpiredFn	IsExpired,
		DenseSet< const MachineBasicBlock * > &	Visited,
		GCNHazardRecognizer::GetNumWaitStatesFn	GetNumWaitStates = SIInstrInfo::getNumWaitStates )

static

Definition at line 568 of file GCNHazardRecognizer.cpp.

References E(), llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and MBB.

Referenced by getWaitStatesSince(), and getWaitStatesSince().

◆ getWaitStatesSince() [2/2]

int getWaitStatesSince	(	GCNHazardRecognizer::IsHazardFn	IsHazard,
		const MachineInstr *	MI,
		GCNHazardRecognizer::IsExpiredFn	IsExpired,
		GCNHazardRecognizer::GetNumWaitStatesFn	GetNumWaitStates = SIInstrInfo::getNumWaitStates )

static

Definition at line 607 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), and MI.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int NumPasses )

static

Definition at line 2684 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates ( int NumPasses )

static

Definition at line 2693 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates ( int NumPasses )

static

Definition at line 3031 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates ( int NumPasses )

static

Definition at line 3003 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2673 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2701 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 2662 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 3021 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates	(	int	NumPasses,
		bool	IsGFX950 )

static

Definition at line 3011 of file GCNHazardRecognizer.cpp.

◆ hasHazard()

template<typename StateT>

bool hasHazard	(	StateT	InitialState,
		function_ref< HazardFnResult(StateT &, const MachineInstr &)>	IsHazard,
		function_ref< void(StateT &, const MachineInstr &)>	UpdateState,
		const MachineBasicBlock *	InitialMBB,
		MachineBasicBlock::const_reverse_instr_iterator	InitialI )

static

Definition at line 464 of file GCNHazardRecognizer.cpp.

References E(), llvm::SmallVectorImpl< T >::emplace_back(), HazardExpired, HazardFound, I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert_as(), isEqual(), LHS, MBB, RHS, llvm::SetVector< T, Vector, Set, N >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

◆ insertNoopsInBundle()

void insertNoopsInBundle	(	MachineInstr *	MI,
		const SIInstrInfo &	TII,
		unsigned	Quantity )

static

Definition at line 270 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), MI, and TII.

◆ isCoexecutableVALUInst()

bool isCoexecutableVALUInst ( const MachineInstr & MI )

static

Definition at line 2041 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isTRANS(), llvm::SIInstrInfo::isVALU(), llvm::SIInstrInfo::isWMMA(), and MI.

◆ isDivFMas()

bool isDivFMas ( unsigned Opcode )

static

Definition at line 95 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isLdsDma()

bool isLdsDma ( const MachineInstr & MI )

static

Definition at line 178 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isMUBUF(), llvm::SIInstrInfo::isVALU(), and MI.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isPermlane()

bool isPermlane ( const MachineInstr & MI )

static

Definition at line 160 of file GCNHazardRecognizer.cpp.

References MI.

Referenced by llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRFE()

bool isRFE ( unsigned Opcode )

static

Definition at line 118 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRWLane()

bool isRWLane ( unsigned Opcode )

static

Definition at line 114 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSendMsgTraceDataOrGDS()

bool isSendMsgTraceDataOrGDS	(	const SIInstrInfo &	TII,
		const MachineInstr &	MI )

static

Definition at line 134 of file GCNHazardRecognizer.cpp.

References MI, and TII.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSGetReg()

bool isSGetReg ( unsigned Opcode )

static

Definition at line 99 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSMovRel()

bool isSMovRel ( unsigned Opcode )

static

Definition at line 122 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSSetReg()

bool isSSetReg ( unsigned Opcode )

static

Definition at line 103 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isStoreCountWaitZero()

bool isStoreCountWaitZero ( const MachineInstr & I )

static

Definition at line 1553 of file GCNHazardRecognizer.cpp.

References I.

◆ isVCmpXWritesExec()

bool isVCmpXWritesExec	(	const SIInstrInfo &	TII,
		const SIRegisterInfo &	TRI,
		const MachineInstr &	MI )

static

Definition at line 1313 of file GCNHazardRecognizer.cpp.

References MI, TII, and TRI.

◆ IsWMMAHazardInstInCategory()

bool IsWMMAHazardInstInCategory	(	const MachineInstr &	MI,
		const SIInstrInfo *	TII,
		unsigned	Latency,
		unsigned	Category )

static

Definition at line 2046 of file GCNHazardRecognizer.cpp.

References assert(), llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isWMMA(), llvm::Latency, MI, and TII.

◆ shouldRunLdsBranchVmemWARHazardFixup()

bool shouldRunLdsBranchVmemWARHazardFixup	(	const MachineFunction &	MF,
		const GCNSubtarget &	ST )

static

Definition at line 1533 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isDS(), llvm::SIInstrInfo::isLDSDMA(), llvm::SIInstrInfo::isVMEM(), MBB, and MI.

Referenced by llvm::GCNHazardRecognizer::GCNHazardRecognizer().

◆ STATISTIC() [1/2]

STATISTIC	(	NumWMMAHoistingBailed	,
		"Number of WMMA hazards where V_NOP hoisting was not possible"	)

◆ STATISTIC() [2/2]

STATISTIC	(	NumWMMANopsHoisted	,
		"Number of WMMA hazard V_NOPs hoisted from loops"	)

◆ updateGetPCBundle()

void updateGetPCBundle ( MachineInstr * NewMI )

static

Definition at line 3361 of file GCNHazardRecognizer.cpp.

References assert(), llvm::MachineBasicBlock::end(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getParent(), I, and llvm::MachineInstr::isBundled().

Variable Documentation

◆ EnableWMMAVnopHoisting

cl::opt< bool > EnableWMMAVnopHoisting("amdgpu-wmma-vnop-hoisting", cl::init(true), cl::Hidden, cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"))	(	"amdgpu-wmma-vnop-hoisting"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders")	)

static

◆ MFMAPaddingRatio

cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))	(	"amdgpu-mfma-padding-ratio"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")	)

static

◆ NopPadding

cl::opt< unsigned > NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction"))	(	"amdgpu-snop-padding"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Insert a s_nop x before every instruction")	)

static

Referenced by llvm::GCNHazardRecognizer::PreEmitNoops().

Macros

Enumerations

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

Enumeration Type Documentation

◆ HazardFnResult

Function Documentation

◆ addRegsToSet()

◆ addRegUnits()

◆ breaksSMEMSoftClause()

◆ breaksVMEMSoftClause()

◆ consumesDstSelForwardingOperand()

◆ ensureEntrySetPrio()

◆ getDstSelForwardingOperand()

◆ getHWReg()

◆ getWaitStatesSince() [1/2]

◆ getWaitStatesSince() [2/2]

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

◆ GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

◆ hasHazard()

◆ insertNoopsInBundle()

◆ isCoexecutableVALUInst()

◆ isDivFMas()

◆ isLdsDma()

◆ isPermlane()

◆ isRFE()

◆ isRWLane()

◆ isSendMsgTraceDataOrGDS()

◆ isSGetReg()

◆ isSMovRel()

◆ isSSetReg()

◆ isStoreCountWaitZero()

◆ isVCmpXWritesExec()

◆ IsWMMAHazardInstInCategory()

◆ shouldRunLdsBranchVmemWARHazardFixup()

◆ STATISTIC() [1/2]

◆ STATISTIC() [2/2]

◆ updateGetPCBundle()

Variable Documentation

◆ EnableWMMAVnopHoisting

◆ MFMAPaddingRatio

◆ NopPadding