SIInsertWaitcnts.cpp File Reference

Insert wait instructions for memory reads and writes. More...

#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/TargetParser/TargetParser.h"

Go to the source code of this file.

Classes
struct	llvm::enum_iteration_traits< InstCounterType >

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "si-insert-waitcnts"

Functions
	DEBUG_COUNTER (ForceExpCounter, DEBUG_TYPE"-forceexp", "Force emit s_waitcnt expcnt(0) instrs")
	DEBUG_COUNTER (ForceLgkmCounter, DEBUG_TYPE"-forcelgkm", "Force emit s_waitcnt lgkmcnt(0) instrs")
	DEBUG_COUNTER (ForceVMCounter, DEBUG_TYPE"-forcevm", "Force emit s_waitcnt vmcnt(0) instrs")
	INITIALIZE_PASS_BEGIN (SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false, false) INITIALIZE_PASS_END(SIInsertWaitcnts
static bool	updateOperandIfDifferent (MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static std::optional< InstCounterType >	counterTypeForInstr (unsigned Opcode)
	Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is waiting on.
static bool	readsVCCZ (const MachineInstr &MI)
static bool	callWaitsOnFunctionEntry (const MachineInstr &MI)
static bool	callWaitsOnFunctionReturn (const MachineInstr &MI)
static bool	isCacheInvOrWBInst (MachineInstr &Inst)
static bool	isWaitInstr (MachineInstr &Inst)

Variables
static cl::opt< bool >	ForceEmitZeroFlag ("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
	DEBUG_TYPE
SI Insert	Waitcnts
SI Insert	false

Detailed Description

Insert wait instructions for memory reads and writes.

Memory reads and writes are issued asynchronously, so we need to insert S_WAITCNT instructions when we want to access any of their results or overwrite any register that's used asynchronously.

TODO: This pass currently keeps one timeline per hardware counter. A more finely-grained approach that keeps one timeline per event type could sometimes get away with generating weaker s_waitcnt instructions. For example, when both SMEM and LDS are in flight and we need to wait for the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient, but the pass will currently generate a conservative lgkmcnt(0) because multiple event types are in flight.

Definition in file SIInsertWaitcnts.cpp.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "si-insert-waitcnts"

Definition at line 42 of file SIInsertWaitcnts.cpp.

Function Documentation

callWaitsOnFunctionEntry()

static bool callWaitsOnFunctionEntry ( const MachineInstr &  MI )

static

Returns

true if the callee inserts an s_waitcnt 0 on function entry.

Definition at line 1586 of file SIInsertWaitcnts.cpp.

callWaitsOnFunctionReturn()

static bool callWaitsOnFunctionReturn ( const MachineInstr &  MI )

static

Returns

true if the callee is expected to wait for any outstanding waits before returning.

Definition at line 1596 of file SIInsertWaitcnts.cpp.

counterTypeForInstr()

static std::optional< InstCounterType > counterTypeForInstr ( unsigned  Opcode )

static

Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is waiting on.

Definition at line 1150 of file SIInsertWaitcnts.cpp.

Referenced by isWaitInstr().

DEBUG_COUNTER() [1/3]

DEBUG_COUNTER	(	ForceExpCounter	,
		DEBUG_TYPE"-forceexp"	,
		"Force emit s_waitcnt expcnt(0) instrs"
	)

DEBUG_COUNTER() [2/3]

DEBUG_COUNTER	(	ForceLgkmCounter	,
		DEBUG_TYPE"-forcelgkm"	,
		"Force emit s_waitcnt lgkmcnt(0) instrs"
	)

DEBUG_COUNTER() [3/3]

DEBUG_COUNTER	(	ForceVMCounter	,
		DEBUG_TYPE"-forcevm"	,
		"Force emit s_waitcnt vmcnt(0) instrs"
	)

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	SIInsertWaitcnts	,
		DEBUG_TYPE	,
		"SI Insert Waitcnts"	,
		false	,
		false
	)

isCacheInvOrWBInst()

static bool isCacheInvOrWBInst ( MachineInstr &  Inst )

static

Definition at line 1974 of file SIInsertWaitcnts.cpp.

References llvm::MachineInstr::getOpcode().

isWaitInstr()

static bool isWaitInstr ( MachineInstr &  Inst )

static

Definition at line 2141 of file SIInsertWaitcnts.cpp.

References counterTypeForInstr(), llvm::SIInstrInfo::getNonSoftWaitcntOpcode(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and llvm::MachineOperand::isReg().

readsVCCZ()

static bool readsVCCZ ( const MachineInstr &  MI )

static

Definition at line 1579 of file SIInsertWaitcnts.cpp.

References MI.

updateOperandIfDifferent()

static bool updateOperandIfDifferent ( MachineInstr &  MI, uint16_t  OpName, unsigned  NewEnc  )

static

Definition at line 1134 of file SIInsertWaitcnts.cpp.

References assert(), llvm::MachineOperand::getImm(), llvm::AMDGPU::getNamedOperandIdx(), MI, and llvm::MachineOperand::setImm().

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 1123 of file SIInsertWaitcnts.cpp.

false

SI Insert false

Definition at line 1123 of file SIInsertWaitcnts.cpp.

ForceEmitZeroFlag

cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden) ( "amdgpu-waitcnt-forcezero"  , cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)")  , cl::init(false)  , cl::Hidden    )

static

Waitcnts

SI Insert Waitcnts

Definition at line 1123 of file SIInsertWaitcnts.cpp.