46#define DEBUG_TYPE "si-insert-waitcnts"
50template <
typename EmitWaitcntFn>
51static void EmitExpandedWaitcnt(
unsigned Outstanding,
unsigned Target,
52 EmitWaitcntFn &&EmitWaitcnt) {
54 for (
unsigned I = Outstanding - 1;
I >
Target &&
I != ~0u; --
I)
74 TRACKINGID_RANGE_LEN = (1 << 16),
79 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
84 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
85 LDSDMA_BEGIN = REGUNITS_END,
86 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
90static constexpr VMEMID toVMEMID(MCRegUnit RU) {
91 return static_cast<unsigned>(RU);
103 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
104 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
105 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
106 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
107 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
112 switch (
MI.getOpcode()) {
113 case AMDGPU::ASYNCMARK:
114 case AMDGPU::WAIT_ASYNCMARK:
117 return MI.isMetaInstruction();
132class WaitcntBrackets;
140class WaitcntGenerator {
142 const GCNSubtarget &ST;
143 const SIInstrInfo &TII;
144 AMDGPU::IsaVersion IV;
147 bool ExpandWaitcntProfiling =
false;
148 const AMDGPU::HardwareLimits &Limits;
152 DenseSet<MachineInstr *> DeferredErasures;
155 WaitcntGenerator() =
delete;
156 WaitcntGenerator(
const WaitcntGenerator &) =
delete;
159 const AMDGPU::HardwareLimits &Limits)
160 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
164 ExpandWaitcntProfiling(
165 MF.
getFunction().hasFnAttribute(
"amdgpu-expand-waitcnt-profiling")),
170 bool isOptNone()
const {
return OptNone; }
186 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
187 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &
Wait,
190 bool eraseDeferredWaitcnts() {
192 for (MachineInstr *
MI : DeferredErasures) {
193 MI->eraseFromParent();
196 DeferredErasures.clear();
201 bool promoteSoftWaitCnt(MachineInstr *Waitcnt)
const;
206 virtual bool createNewWaitcnt(MachineBasicBlock &
Block,
208 AMDGPU::Waitcnt
Wait,
209 const WaitcntBrackets &ScoreBrackets) = 0;
216 assert(
E.size() == 1 &&
"Cannot handle a mask of events!");
218 if (getWaitEvents(
T) &
E)
229 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const = 0;
231 virtual ~WaitcntGenerator() =
default;
234class WaitcntGeneratorPreGFX12 final :
public WaitcntGenerator {
238 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
239 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS |
240 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
242 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
243 HWEvents::SQ_MESSAGE,
245 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
246 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
247 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
256 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
257 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
259 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
260 HWEvents::SQ_MESSAGE,
262 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
263 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
264 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
266 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
272 using WaitcntGenerator::WaitcntGenerator;
273 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
274 MachineInstr &OldWaitcntInstr,
275 AMDGPU::Waitcnt &
Wait,
278 bool createNewWaitcnt(MachineBasicBlock &
Block,
280 AMDGPU::Waitcnt
Wait,
281 const WaitcntBrackets &ScoreBrackets)
override;
284 return ST.hasVscnt() ? WaitEventMaskForInstWithVscnt[
T]
285 : WaitEventMaskForInstWithoutVscnt[
T];
288 AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
291class WaitcntGeneratorGFX12Plus final :
public WaitcntGenerator {
296 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
297 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
298 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
299 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
300 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
302 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
303 HWEvents::VMEM_SAMPLER_READ_ACCESS,
304 HWEvents::VMEM_BVH_READ_ACCESS,
306 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
307 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
308 HWEvents::ASYNC_ACCESS,
309 HWEvents::TENSOR_ACCESS,
310 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
311 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
312 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
313 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
314 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
315 HWEvents::VGPR_VMEM_READ};
318 WaitcntGeneratorGFX12Plus() =
delete;
321 const AMDGPU::HardwareLimits &Limits,
323 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
325 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
326 MachineInstr &OldWaitcntInstr,
327 AMDGPU::Waitcnt &
Wait,
330 bool createNewWaitcnt(MachineBasicBlock &
Block,
332 AMDGPU::Waitcnt
Wait,
333 const WaitcntBrackets &ScoreBrackets)
override;
336 return WaitEventMaskForInstGFX12Plus[
T];
339 AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
343struct PreheaderFlushFlags {
344 bool FlushVmCnt =
false;
345 bool FlushDsCnt =
false;
348class SIInsertWaitcnts {
349 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
350 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
351 MachineLoopInfo &MLI;
352 MachinePostDominatorTree &PDT;
357 std::unique_ptr<WaitcntBrackets> Incoming;
359 BlockInfo() =
default;
360 BlockInfo(BlockInfo &&) =
default;
361 BlockInfo &operator=(BlockInfo &&) =
default;
365 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
369 std::unique_ptr<WaitcntGenerator> WCG;
372 DenseSet<MachineInstr *> CallInsts;
373 DenseSet<MachineInstr *> ReturnInsts;
378 DenseMap<MachineInstr *, bool> EndPgmInsts;
380 AMDGPU::HardwareLimits Limits;
383 const GCNSubtarget &ST;
384 const SIInstrInfo &TII;
385 const SIRegisterInfo &TRI;
386 const MachineRegisterInfo &MRI;
389 bool IsExpertMode =
false;
392 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
394 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
395 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
396 MRI(MF.getRegInfo()),
397 TgSplit(ST.hasTgSplitSupport() &&
400 const AMDGPU::HardwareLimits &getLimits()
const {
return Limits; }
402 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *
ML,
403 const WaitcntBrackets &Brackets);
404 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &
MBB,
405 const WaitcntBrackets &ScoreBrackets);
406 bool isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const;
407 bool isDSRead(
const MachineInstr &
MI)
const;
408 bool mayStoreIncrementingDSCNT(
const MachineInstr &
MI)
const;
411 bool isVmemAccess(
const MachineInstr &
MI)
const;
412 bool generateWaitcntInstBefore(MachineInstr &
MI,
413 WaitcntBrackets &ScoreBrackets,
414 MachineInstr *OldWaitcntInstr,
415 PreheaderFlushFlags FlushFlags);
416 bool generateWaitcnt(AMDGPU::Waitcnt
Wait,
418 MachineBasicBlock &
Block, WaitcntBrackets &ScoreBrackets,
419 MachineInstr *OldWaitcntInstr);
420 void updateEventWaitcntAfter(MachineInstr &Inst,
421 WaitcntBrackets *ScoreBrackets);
423 MachineBasicBlock *
Block)
const;
424 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &
Block,
425 WaitcntBrackets &ScoreBrackets);
427 WaitcntBrackets &ScoreBrackets);
430 bool removeRedundantSoftXcnts(MachineBasicBlock &
Block);
432 bool ExpertMode)
const;
434 return WCG->getWaitEvents(
T);
437 return WCG->getCounterFromEvent(
E);
449class WaitcntBrackets {
451 WaitcntBrackets(
const SIInsertWaitcnts *Context) : Context(Context) {
452 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
457 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
458 for (
auto &[ID, Val] : VMem) {
462 for (
auto &[ID, Val] : SGPRs) {
467 if (NumUnusedVmem || NumUnusedSGPRs) {
468 errs() <<
"WaitcntBracket had unused entries at destruction time: "
469 << NumUnusedVmem <<
" VMem and " << NumUnusedSGPRs
470 <<
" SGPR unused entries\n";
481 return ScoreUBs[
T] - ScoreLBs[
T];
485 return getVMemScore(ID,
T) > getScoreLB(
T);
503 return getScoreUB(
T) - getScoreLB(
T);
507 auto It = SGPRs.find(RU);
508 return It != SGPRs.end() ? It->second.get(
T) : 0;
512 auto It = VMem.find(TID);
513 return It != VMem.end() ? It->second.Scores[
T] : 0;
518 unsigned getNumOrderedXDLsAfterWrite(VMEMID TID)
const {
519 return std::min(OrderedXDLCount - VMem.at(TID).OrderedXDLCountAtWrite,
535 void simplifyWaitcnt(AMDGPU::Waitcnt &
Wait)
const {
538 void simplifyWaitcnt(
const AMDGPU::Waitcnt &CheckWait,
539 AMDGPU::Waitcnt &UpdateWait)
const;
542 void simplifyXcnt(
const AMDGPU::Waitcnt &CheckWait,
543 AMDGPU::Waitcnt &UpdateWait)
const;
544 void simplifyVmVsrc(
const AMDGPU::Waitcnt &CheckWait,
545 AMDGPU::Waitcnt &UpdateWait)
const;
548 void simplifyVmemInvOnlyWait(AMDGPU::Waitcnt &
Wait)
const {
552 if (LoadEvents == HWEvents::VMEM_INV_ACCESS)
557 AMDGPU::Waitcnt &
Wait,
558 const MachineInstr &
MI)
const;
559 MCRegister determineVGPR16Dependency(
const MachineInstr &
MI,
561 MCRegister
Reg)
const;
563 AMDGPU::Waitcnt &
Wait)
const;
565 void tryClearSCCWriteEvent(MachineInstr *Inst);
567 void applyWaitcnt(
const AMDGPU::Waitcnt &
Wait);
573 HWEvents getPendingEvents()
const {
return PendingEvents; }
574 bool hasPendingEvent()
const {
return PendingEvents.
any(); }
575 bool hasPendingEvent(
HWEvents E)
const {
return PendingEvents.contains(
E); }
577 bool HasPending = (PendingEvents & Context->getWaitEvents(
T)).any();
579 "Expected pending events iff scoreboard is not empty");
584 HWEvents Events = PendingEvents & Context->getWaitEvents(
T);
586 return Events.
size() > 1;
589 bool hasPendingFlat()
const {
596 void setPendingFlat() {
601 bool hasPendingXcntOnlyVmem()
const {
606 void setPendingXcntOnlyVmem() { LastXcntOnlyVmem = ScoreUBs[
AMDGPU::X_CNT]; }
611 bool hasPendingNonAsyncXcnt(
612 const AMDGPU::Waitcnt &
Wait = AMDGPU::Waitcnt())
const {
613 if (hasPendingXcntOnlyVmem() || (hasPendingEvent(HWEvents::SMEM_GROUP) &&
618 if (
Wait.get(
T) != 0 && hasPendingEvent(
T))
624 bool hasPendingGDS()
const {
629 unsigned getPendingGDSWait()
const {
638 bool hasDifferentVGPRPendingEvents(MCRegister
Reg,
HWEvents E)
const {
639 for (MCRegUnit RU : regunits(
Reg)) {
640 auto It = VMem.find(toVMEMID(RU));
641 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~
E).any())
647 void clearVGPRPendingEvents(MCRegister
Reg) {
648 for (MCRegUnit RU : regunits(
Reg)) {
649 if (
auto It = VMem.find(toVMEMID(RU)); It != VMem.end()) {
651 if (It->second.empty())
657 void setStateOnFunctionEntryOrReturn() {
663 ArrayRef<const MachineInstr *> getLDSDMAStores()
const {
667 bool hasPointSampleAccel(
const MachineInstr &
MI)
const;
668 bool hasPointSamplePendingVmemTypes(
const MachineInstr &
MI,
669 MCRegister RU)
const;
671 void print(raw_ostream &)
const;
676 void purgeEmptyTrackingData();
680 return Context->getLimits().get(
T);
690 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
694 AMDGPU::Waitcnt &
Wait)
const;
695 void determineWaitForVAVDSTWrite(VMEMID ID, AMDGPU::Waitcnt &
Wait)
const;
697 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
698 unsigned OtherScore);
704 assert(
Reg != AMDGPU::SCC &&
"Shouldn't be used on SCC");
705 if (!Context->TRI.isInAllocatableClass(
Reg))
707 return Context->TRI.regunits(
Reg);
728 const SIRegisterInfo &
TRI = Context->TRI;
729 if (
Reg == AMDGPU::SCC) {
731 }
else if (
TRI.isVectorRegister(Context->MRI,
Reg)) {
732 for (MCRegUnit RU : regunits(
Reg)) {
733 auto &
Info = VMem[toVMEMID(RU)];
734 Info.Scores[
T] = Val;
736 Info.OrderedXDLCountAtWrite = OrderedXDLCount;
738 }
else if (
TRI.isSGPRReg(Context->MRI,
Reg)) {
739 for (MCRegUnit RU : regunits(
Reg))
740 SGPRs[RU].get(
T) = Val;
747 VMem[TID].Scores[
T] = Val;
750 void setScoreByOperand(
const MachineOperand &
Op,
753 const SIInsertWaitcnts *Context;
759 unsigned LastFlatDsCnt = 0;
760 unsigned LastFlatLoadCnt = 0;
764 unsigned LastXcntOnlyVmem = 0;
766 unsigned LastGDS = 0;
769 unsigned OrderedXDLCount = 0;
786 CounterValueArray Scores{};
789 unsigned OrderedXDLCountAtWrite = 0;
803 unsigned ScoreDsKmCnt = 0;
804 unsigned ScoreXCnt = 0;
820 bool empty()
const {
return !ScoreDsKmCnt && !ScoreXCnt; }
823 DenseMap<VMEMID, VMEMInfo> VMem;
824 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
827 unsigned SCCScore = 0;
829 const MachineInstr *PendingSCCWrite =
nullptr;
833 SmallVector<const MachineInstr *> LDSDMAStores;
843 static constexpr unsigned MaxAsyncMarks = 16;
847 std::array<CounterValueArray, AMDGPU::NumAsyncStages> AsyncScore{};
850SIInsertWaitcnts::BlockInfo::~BlockInfo() =
default;
855 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
859 StringRef getPassName()
const override {
860 return "SI insert wait instructions";
863 void getAnalysisUsage(AnalysisUsage &AU)
const override {
866 AU.
addRequired<MachinePostDominatorTreeWrapperPass>();
878 setRegScore(
Op.getReg().asMCReg(), CntTy, Score);
886bool WaitcntBrackets::hasPointSampleAccel(
const MachineInstr &
MI)
const {
891 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
901bool WaitcntBrackets::hasPointSamplePendingVmemTypes(
const MachineInstr &
MI,
902 MCRegister
Reg)
const {
903 if (!hasPointSampleAccel(
MI))
906 return hasDifferentVGPRPendingEvents(
Reg, HWEvents::VMEM_READ_ACCESS);
909void WaitcntBrackets::updateByEvent(
HWEvents E, MachineInstr &Inst) {
910 assert(
E.size() == 1 &&
"Expected singular event!");
914 unsigned UB = getScoreUB(
T);
918 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
930 setScoreUB(
T, CurrScore);
935 if (
E == HWEvents::VGPR_XDL_WRITE &&
Context->TII.isVAVDSTOrderedXDL(Inst))
939 const MachineRegisterInfo &
MRI =
Context->MRI;
948 if (
const auto *AddrOp =
TII.getNamedOperand(Inst, AMDGPU::OpName::addr))
952 if (
const auto *Data0 =
953 TII.getNamedOperand(Inst, AMDGPU::OpName::data0))
955 if (
const auto *Data1 =
956 TII.getNamedOperand(Inst, AMDGPU::OpName::data1))
960 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
961 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
962 for (
const MachineOperand &
Op : Inst.
all_uses()) {
963 if (
TRI.isVectorRegister(
MRI,
Op.getReg()))
967 }
else if (
TII.isFLAT(Inst)) {
969 setScoreByOperand(*
TII.getNamedOperand(Inst, AMDGPU::OpName::data),
972 setScoreByOperand(*
TII.getNamedOperand(Inst, AMDGPU::OpName::data),
975 }
else if (
TII.isMIMG(Inst)) {
979 setScoreByOperand(*
TII.getNamedOperand(Inst, AMDGPU::OpName::data),
982 }
else if (
TII.isMTBUF(Inst)) {
985 }
else if (
TII.isMUBUF(Inst)) {
989 setScoreByOperand(*
TII.getNamedOperand(Inst, AMDGPU::OpName::data),
992 }
else if (
TII.isLDSDIR(Inst)) {
994 setScoreByOperand(*
TII.getNamedOperand(Inst, AMDGPU::OpName::vdst),
997 if (
TII.isEXP(Inst)) {
1002 for (MachineOperand &DefMO : Inst.
all_defs()) {
1003 if (
TRI.isVGPR(
MRI, DefMO.getReg())) {
1008 for (
const MachineOperand &
Op : Inst.
all_uses()) {
1009 if (
TRI.isVectorRegister(
MRI,
Op.getReg()))
1015 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
1016 if (PendingEvents.
contains(OtherEvent)) {
1021 setScoreLB(
T, getScoreUB(
T) - 1);
1022 PendingEvents -= OtherEvent;
1024 for (
const MachineOperand &
Op : Inst.
all_uses()) {
1027 if (
Op.getReg() == AMDGPU::ASYNCcnt)
1029 setScoreByOperand(
Op,
T, CurrScore);
1035 for (
const MachineOperand &
Op : Inst.
operands()) {
1048 setScoreByOperand(
Op,
T, CurrScore);
1060 for (
const MachineOperand &
Op : Inst.
defs()) {
1063 if (!
TRI.isVectorRegister(
MRI,
Op.getReg()))
1065 if (updateVMCntOnly(Inst)) {
1074 if (hasPointSampleAccel(Inst))
1075 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1076 for (MCRegUnit RU : regunits(
Op.getReg().asMCReg()))
1077 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1080 setScoreByOperand(
Op,
T, CurrScore);
1083 (
TII.isDS(Inst) ||
TII.isNonAsyncLdsDmaWrite(Inst))) {
1092 if (!MemOp->isStore() ||
1097 auto AAI = MemOp->getAAInfo();
1103 if (!AAI || !AAI.Scope)
1105 for (
size_t I = 0,
E = LDSDMAStores.
size();
I !=
E && !Slot; ++
I) {
1106 for (
const auto *MemOp : LDSDMAStores[
I]->memoperands()) {
1107 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1122 setVMemScore(LDSDMA_BEGIN,
T, CurrScore);
1123 if (Slot && Slot < NUM_LDSDMA)
1124 setVMemScore(LDSDMA_BEGIN +
static_cast<VMEMID
>(Slot),
T, CurrScore);
1132 setRegScore(AMDGPU::SCC,
T, CurrScore);
1133 PendingSCCWrite = &Inst;
1138void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
1148 LLVM_DEBUG(
dbgs() <<
"recordAsyncMark(stages=" << Stages <<
"):\n" << Inst);
1151 AsyncMarks[
SI].push_back(AsyncScore[SI]);
1153 dbgs() <<
" stage " << S <<
":\n";
1154 for (
const auto &Mark : AsyncMarks[SI]) {
1163void WaitcntBrackets::print(raw_ostream &OS)
const {
1167 unsigned SR = getScoreRange(
T);
1170 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"LOAD" :
"VM") <<
"_CNT("
1174 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"DS" :
"LGKM") <<
"_CNT("
1178 OS <<
" EXP_CNT(" << SR <<
"):";
1181 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"STORE" :
"VS") <<
"_CNT("
1185 OS <<
" SAMPLE_CNT(" << SR <<
"):";
1188 OS <<
" BVH_CNT(" << SR <<
"):";
1191 OS <<
" KM_CNT(" << SR <<
"):";
1194 OS <<
" X_CNT(" << SR <<
"):";
1197 OS <<
" ASYNC_CNT(" << SR <<
"):";
1200 OS <<
" VA_VDST_RD(" << SR <<
"): ";
1203 OS <<
" VA_VDST_WR(" << SR <<
"): ";
1206 OS <<
" VM_VSRC(" << SR <<
"): ";
1209 OS <<
" UNKNOWN(" << SR <<
"):";
1215 unsigned LB = getScoreLB(
T);
1218 sort(SortedVMEMIDs);
1220 for (
auto ID : SortedVMEMIDs) {
1221 unsigned RegScore = VMem.at(ID).Scores[
T];
1224 unsigned RelScore = RegScore - LB - 1;
1225 if (ID < REGUNITS_END) {
1226 OS <<
' ' << RelScore <<
':'
1229 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1230 "Unhandled/unexpected ID value!");
1231 OS <<
' ' << RelScore <<
":LDSDMA" <<
ID;
1236 if (isSmemCounter(
T)) {
1238 sort(SortedSMEMIDs);
1239 for (
auto ID : SortedSMEMIDs) {
1240 unsigned RegScore = SGPRs.at(ID).get(
T);
1243 unsigned RelScore = RegScore - LB - 1;
1244 OS <<
' ' << RelScore <<
':'
1250 OS <<
' ' << SCCScore <<
":scc";
1255 OS <<
"Pending Events: ";
1256 if (hasPendingEvent()) {
1257 OS << getPendingEvents();
1265 const CounterValueArray &Score = AsyncScore[
SI];
1266 OS <<
"Async score (stage " << S <<
"): ";
1273 OS <<
"Async marks (stage " << S <<
"): " << AsyncMarks[
SI].size() <<
'\n';
1275 for (
const auto &Mark : AsyncMarks[SI]) {
1277 unsigned MarkedScore = Mark[
T];
1280 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"LOAD" :
"VM")
1281 <<
"_CNT: " << MarkedScore;
1284 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"DS" :
"LGKM")
1285 <<
"_CNT: " << MarkedScore;
1288 OS <<
" EXP_CNT: " << MarkedScore;
1291 OS <<
" " << (
ST.hasExtendedWaitCounts() ?
"STORE" :
"VS")
1292 <<
"_CNT: " << MarkedScore;
1295 OS <<
" SAMPLE_CNT: " << MarkedScore;
1298 OS <<
" BVH_CNT: " << MarkedScore;
1301 OS <<
" KM_CNT: " << MarkedScore;
1304 OS <<
" X_CNT: " << MarkedScore;
1307 OS <<
" ASYNC_CNT: " << MarkedScore;
1310 OS <<
" UNKNOWN: " << MarkedScore;
1322void WaitcntBrackets::simplifyWaitcnt(
const AMDGPU::Waitcnt &CheckWait,
1323 AMDGPU::Waitcnt &UpdateWait)
const {
1331 simplifyXcnt(CheckWait, UpdateWait);
1334 simplifyVmVsrc(CheckWait, UpdateWait);
1339 unsigned &
Count)
const {
1343 if (
Count >= getScoreRange(
T))
1347void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &
Wait,
1349 unsigned Cnt =
Wait.get(
T);
1350 simplifyWaitcnt(
T, Cnt);
1354void WaitcntBrackets::simplifyXcnt(
const AMDGPU::Waitcnt &CheckWait,
1355 AMDGPU::Waitcnt &UpdateWait)
const {
1365 hasPendingEvent(HWEvents::SMEM_GROUP))
1372 hasPendingEvent(HWEvents::VMEM_GROUP) &&
1374 !hasPendingEvent(HWEvents::ASYNC_ACCESS) &&
1379 hasPendingEvent(HWEvents::ASYNC_ACCESS) &&
1380 !hasPendingNonAsyncXcnt(CheckWait))
1385void WaitcntBrackets::simplifyVmVsrc(
const AMDGPU::Waitcnt &CheckWait,
1386 AMDGPU::Waitcnt &UpdateWait)
const {
1396 return Acc |
Context->getWaitEvents(
T);
1398 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1400 unsigned CheckCount = CheckWait.
get(
T);
1402 (CheckCount == 0 || !counterOutOfOrder(
T)) &&
1403 (PendingVmemEvents & ~
Context->getWaitEvents(
T)) == 0)
1410void WaitcntBrackets::purgeEmptyTrackingData() {
1411 VMem.remove_if([](
const auto &
P) {
return P.second.empty(); });
1412 SGPRs.remove_if([](
const auto &
P) {
return P.second.empty(); });
1417 unsigned ScoreToWait)
const {
1418 const unsigned LB = getScoreLB(
T);
1419 const unsigned UB = getScoreUB(
T);
1422 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1424 !
Context->ST.hasFlatLgkmVMemCountInOrder()) {
1430 if (counterOutOfOrder(
T)) {
1438 return std::min(UB - ScoreToWait, getLimit(
T) - 1);
1444 unsigned ScoreToWait,
1445 AMDGPU::Waitcnt &
Wait)
const {
1446 Wait.add(
T, getWaitForScore(
T, ScoreToWait));
1449void WaitcntBrackets::determineWaitForVAVDSTWrite(VMEMID ID,
1450 AMDGPU::Waitcnt &
Wait)
const {
1460 std::max(
Count, getNumOrderedXDLsAfterWrite(ID)));
1468 AMDGPU::Waitcnt
Wait;
1472 dbgs() <<
"Need " <<
N <<
" async marks in stage " << S <<
". Found "
1473 << StageMarks.size() <<
":\n";
1474 for (
const auto &Mark : StageMarks) {
1480 unsigned StageN =
N;
1481 if (StageMarks.size() == MaxAsyncMarks) {
1487 dbgs() <<
"Possible truncation. Ensuring a non-trivial wait.\n");
1488 StageN = std::min(StageN, (
unsigned)MaxAsyncMarks - 1);
1491 if (StageMarks.size() <= StageN) {
1496 size_t MarkIndex = StageMarks.size() - StageN - 1;
1497 const auto &RequiredMark = StageMarks[MarkIndex];
1499 determineWaitForScore(
T, RequiredMark[
T],
Wait);
1506 dbgs() <<
"Removing " << (MarkIndex + 1)
1507 <<
" async marks after determining wait\n";
1509 StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
1523MCRegister WaitcntBrackets::determineVGPR16Dependency(
const MachineInstr &
MI,
1525 MCRegister
Reg)
const {
1529 if (
Size != 16 || !
Context->ST.hasD16Writes32BitVgpr())
1539 AMDGPU::Waitcnt
Wait;
1540 for (MCRegUnit RU : regunits(OtherHalf)) {
1542 determineWaitForVAVDSTWrite(toVMEMID(RU),
Wait);
1544 determineWaitForScore(
T, getVMemScore(toVMEMID(RU),
T),
Wait);
1548 if (!
Wait.hasWait())
1558 HWEvents Events = MIEvents & OtherHalfEvents;
1559 if (Events.
size() > 1)
1566 AMDGPU::Waitcnt &
Wait,
1567 const MachineInstr &
MI)
const {
1568 if (
Reg == AMDGPU::SCC) {
1569 determineWaitForScore(
T, SCCScore,
Wait);
1573 Reg = determineVGPR16Dependency(
MI,
T,
Reg);
1574 for (MCRegUnit RU : regunits(
Reg)) {
1576 determineWaitForVAVDSTWrite(toVMEMID(RU),
Wait);
1578 determineWaitForScore(
1579 T, IsVGPR ? getVMemScore(toVMEMID(RU),
T) : getSGPRScore(RU,
T),
1588 AMDGPU::Waitcnt &
Wait)
const {
1589 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1590 determineWaitForScore(
T, getVMemScore(TID,
T),
Wait);
1593void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1596 if (PendingSCCWrite &&
1597 PendingSCCWrite->
getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1599 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1602 SCC_WRITE_PendingEvent) {
1606 PendingEvents -= SCC_WRITE_PendingEvent;
1607 PendingSCCWrite =
nullptr;
1611void WaitcntBrackets::applyWaitcnt(
const AMDGPU::Waitcnt &
Wait) {
1620 applyWaitcnt(
T, Cnt);
1625 const unsigned UB = getScoreUB(
T);
1629 if (counterOutOfOrder(
T))
1631 setScoreLB(
T, std::max(getScoreLB(
T), UB -
Count));
1634 PendingEvents -=
Context->getWaitEvents(
T);
1638 hasPendingEvent(HWEvents::SMEM_GROUP)) {
1642 PendingEvents -= HWEvents::SMEM_GROUP;
1646 !hasPendingEvent(HWEvents::ASYNC_ACCESS)) {
1649 else if (
Count == 0)
1650 PendingEvents -= HWEvents::VMEM_GROUP;
1656void WaitcntBrackets::applyWaitcnt(
const AMDGPU::Waitcnt &
Wait,
1658 unsigned Cnt =
Wait.get(
T);
1659 applyWaitcnt(
T, Cnt);
1666 if ((
T ==
Context->SmemAccessCounter &&
1667 hasPendingEvent(HWEvents::SMEM_ACCESS)) ||
1684 static constexpr HWEvents ExtendedImageEvents =
1685 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1686 if (!
Context->ST.hasExtendedWaitCounts() &&
1687 (Events & ExtendedImageEvents).any()) {
1688 Events -= ExtendedImageEvents;
1689 Events |= HWEvents::VMEM_READ_ACCESS;
1695 Events -= HWEvents::VMEM_INV_ACCESS;
1697 return Events.
size() > 1;
1700 return hasMixedPendingEvents(
T);
1710char SIInsertWaitcntsLegacy::ID = 0;
1715 return new SIInsertWaitcntsLegacy();
1720 int OpIdx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
OpName);
1725 if (NewEnc == MO.
getImm())
1732bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt)
const {
1746bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1747 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1749 assert(isNormalMode(MaxCounter));
1752 MachineInstr *WaitcntInstr =
nullptr;
1753 MachineInstr *WaitcntVsCntInstr =
nullptr;
1756 dbgs() <<
"PreGFX12::applyPreexistingWaitcnt at: ";
1758 dbgs() <<
"end of block\n";
1766 if (isNonWaitcntMetaInst(
II)) {
1772 bool TrySimplify = Opcode !=
II.getOpcode() && !OptNone;
1776 if (Opcode == AMDGPU::S_WAITCNT) {
1777 unsigned IEnc =
static_cast<unsigned>(
II.getOperand(0).
getImm());
1780 ScoreBrackets.simplifyWaitcnt(OldWait);
1781 ScoreBrackets.simplifyVmemInvOnlyWait(OldWait);
1787 DeferredErasures.erase(&
II);
1788 II.eraseFromParent();
1790 }
else if (!
Wait.hasWaitExceptStoreCnt() && TrySimplify) {
1794 DeferredErasures.insert(&
II);
1797 DeferredErasures.erase(&
II);
1799 }
else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1802 <<
"Before: " <<
Wait <<
'\n';);
1813 II.eraseFromParent();
1814 }
else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1815 unsigned N =
static_cast<unsigned>(
II.getOperand(0).
getImm());
1817 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
1821 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1822 assert(
II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1824 unsigned OldVSCnt =
static_cast<unsigned>(
1825 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
1831 if (WaitcntVsCntInstr) {
1832 DeferredErasures.erase(&
II);
1833 II.eraseFromParent();
1835 }
else if (!
Wait.hasWaitStoreCnt() && TrySimplify) {
1837 DeferredErasures.insert(&
II);
1839 WaitcntVsCntInstr = &
II;
1840 DeferredErasures.erase(&
II);
1848 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1857 LLVM_DEBUG(It.isEnd() ?
dbgs() <<
"applied pre-existing waitcnt\n"
1858 <<
"New Instr at block end: "
1859 << *WaitcntInstr <<
'\n'
1860 :
dbgs() <<
"applied pre-existing waitcnt\n"
1861 <<
"Old Instr: " << *It
1862 <<
"New Instr: " << *WaitcntInstr <<
'\n');
1865 if (WaitcntVsCntInstr) {
1869 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1875 ?
dbgs() <<
"applied pre-existing waitcnt\n"
1876 <<
"New Instr at block end: " << *WaitcntVsCntInstr
1878 :
dbgs() <<
"applied pre-existing waitcnt\n"
1879 <<
"Old Instr: " << *It
1880 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
1888bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1890 AMDGPU::Waitcnt
Wait,
const WaitcntBrackets &ScoreBrackets) {
1891 assert(isNormalMode(MaxCounter));
1898 if (
Wait.hasWaitExceptStoreCnt()) {
1900 if (ExpandWaitcntProfiling) {
1904 bool AnyOutOfOrder =
false;
1906 unsigned WaitCnt =
Wait.get(CT);
1907 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(CT)) {
1908 AnyOutOfOrder =
true;
1913 if (AnyOutOfOrder) {
1921 unsigned WaitCnt =
Wait.get(CT);
1925 unsigned Outstanding =
1926 std::min(ScoreBrackets.getOutstanding(CT), getLimit(CT) - 1);
1927 EmitExpandedWaitcnt(Outstanding, WaitCnt, [&](
unsigned Count) {
1939 [[maybe_unused]]
auto SWaitInst =
1944 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1945 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1949 if (
Wait.hasWaitStoreCnt()) {
1955 unsigned Outstanding =
1958 EmitExpandedWaitcnt(
1960 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_VSCNT))
1961 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1966 [[maybe_unused]]
auto SWaitInst =
1968 .
addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1973 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1974 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1982WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1983 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt &&
ST.hasVscnt() ? 0 : ~0u);
1987WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1988 unsigned ExpertVal = IsExpertMode ? 0 : ~0
u;
1989 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1991 ~0u , ExpertVal, ExpertVal, ExpertVal);
1998bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1999 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
2001 assert(!isNormalMode(MaxCounter));
2004 MachineInstr *CombinedLoadDsCntInstr =
nullptr;
2005 MachineInstr *CombinedStoreDsCntInstr =
nullptr;
2006 MachineInstr *WaitcntDepctrInstr =
nullptr;
2010 dbgs() <<
"GFX12Plus::applyPreexistingWaitcnt at: ";
2012 dbgs() <<
"end of block\n";
2018 AMDGPU::Waitcnt RequiredWait;
2023 if (isNonWaitcntMetaInst(
II)) {
2032 bool TrySimplify = Opcode !=
II.getOpcode() && !OptNone;
2036 if (Opcode == AMDGPU::S_WAITCNT)
2039 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
2040 unsigned OldEnc =
static_cast<unsigned>(
2041 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
2046 RequiredWait = RequiredWait.combined(OldWait);
2048 if (CombinedLoadDsCntInstr ==
nullptr) {
2049 CombinedLoadDsCntInstr = &
II;
2051 II.eraseFromParent();
2054 }
else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
2055 unsigned OldEnc =
static_cast<unsigned>(
2056 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
2061 RequiredWait = RequiredWait.combined(OldWait);
2063 if (CombinedStoreDsCntInstr ==
nullptr) {
2064 CombinedStoreDsCntInstr = &
II;
2066 II.eraseFromParent();
2069 }
else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
2070 unsigned OldEnc =
static_cast<unsigned>(
2071 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
2072 AMDGPU::Waitcnt OldWait;
2079 ScoreBrackets.simplifyWaitcnt(OldWait);
2081 if (WaitcntDepctrInstr ==
nullptr) {
2082 WaitcntDepctrInstr = &
II;
2090 unsigned Enc =
static_cast<unsigned>(
2091 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
2102 II.eraseFromParent();
2106 }
else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
2109 II.eraseFromParent();
2111 }
else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
2114 unsigned N =
static_cast<unsigned>(
II.getOperand(0).
getImm());
2115 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
2119 std::optional<AMDGPU::InstCounterType> CT =
2122 unsigned OldCnt =
static_cast<unsigned>(
2123 TII.getNamedOperand(
II, AMDGPU::OpName::simm16)->
getImm());
2125 Wait.add(CT.value(), OldCnt);
2127 RequiredWait.add(CT.value(), OldCnt);
2129 if (WaitInstrs[CT.value()] ==
nullptr) {
2130 WaitInstrs[CT.value()] = &
II;
2132 II.eraseFromParent();
2138 ScoreBrackets.simplifyWaitcnt(
Wait.combined(RequiredWait),
Wait);
2139 Wait =
Wait.combined(RequiredWait);
2141 if (CombinedLoadDsCntInstr) {
2157 AMDGPU::OpName::simm16, NewEnc);
2158 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
2164 LLVM_DEBUG(It.isEnd() ?
dbgs() <<
"applied pre-existing waitcnt\n"
2165 <<
"New Instr at block end: "
2166 << *CombinedLoadDsCntInstr <<
'\n'
2167 :
dbgs() <<
"applied pre-existing waitcnt\n"
2168 <<
"Old Instr: " << *It <<
"New Instr: "
2169 << *CombinedLoadDsCntInstr <<
'\n');
2176 if (CombinedStoreDsCntInstr) {
2181 AMDGPU::OpName::simm16, NewEnc);
2182 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
2188 LLVM_DEBUG(It.isEnd() ?
dbgs() <<
"applied pre-existing waitcnt\n"
2189 <<
"New Instr at block end: "
2190 << *CombinedStoreDsCntInstr <<
'\n'
2191 :
dbgs() <<
"applied pre-existing waitcnt\n"
2192 <<
"Old Instr: " << *It <<
"New Instr: "
2193 << *CombinedStoreDsCntInstr <<
'\n');
2223 for (MachineInstr **WI : WaitsToErase) {
2227 DeferredErasures.erase(*WI);
2228 (*WI)->eraseFromParent();
2235 if (!WaitInstrs[CT])
2238 unsigned NewCnt =
Wait.get(CT);
2239 if (NewCnt != ~0u) {
2241 AMDGPU::OpName::simm16, NewCnt);
2242 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
2243 DeferredErasures.erase(WaitInstrs[CT]);
2245 ScoreBrackets.applyWaitcnt(CT, NewCnt);
2249 ?
dbgs() <<
"applied pre-existing waitcnt\n"
2250 <<
"New Instr at block end: " << *WaitInstrs[CT]
2252 :
dbgs() <<
"applied pre-existing waitcnt\n"
2253 <<
"Old Instr: " << *It
2254 <<
"New Instr: " << *WaitInstrs[CT] <<
'\n');
2257 DeferredErasures.insert(WaitInstrs[CT]);
2261 if (WaitcntDepctrInstr) {
2264 unsigned Enc =
static_cast<unsigned>(
2265 TII.getNamedOperand(*WaitcntDepctrInstr, AMDGPU::OpName::simm16)
2285 AMDGPU::OpName::simm16, Enc);
2287 <<
"New Instr at block end: "
2288 << *WaitcntDepctrInstr <<
'\n'
2289 :
dbgs() <<
"applyPreexistingWaitcnt\n"
2290 <<
"Old Instr: " << *It <<
"New Instr: "
2291 << *WaitcntDepctrInstr <<
'\n');
2302bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2304 AMDGPU::Waitcnt
Wait,
const WaitcntBrackets &ScoreBrackets) {
2305 assert(!isNormalMode(MaxCounter));
2312 if (ExpandWaitcntProfiling) {
2319 if (ScoreBrackets.counterOutOfOrder(CT)) {
2326 unsigned Outstanding =
2327 std::min(ScoreBrackets.getOutstanding(CT), getLimit(CT) - 1);
2328 EmitExpandedWaitcnt(Outstanding,
Count, [&](
unsigned Val) {
2340 MachineInstr *SWaitInst =
nullptr;
2364 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
2365 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
2377 [[maybe_unused]]
auto SWaitInst =
2384 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
2385 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
2388 if (
Wait.hasWaitDepctr()) {
2397 [[maybe_unused]]
auto SWaitInst =
2403 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
2404 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
2423bool SIInsertWaitcnts::generateWaitcntInstBefore(
2424 MachineInstr &
MI, WaitcntBrackets &ScoreBrackets,
2425 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2430 AMDGPU::Waitcnt
Wait;
2431 const unsigned Opc =
MI.getOpcode();
2434 case AMDGPU::BUFFER_WBINVL1:
2435 case AMDGPU::BUFFER_WBINVL1_SC:
2436 case AMDGPU::BUFFER_WBINVL1_VOL:
2437 case AMDGPU::BUFFER_GL0_INV:
2438 case AMDGPU::BUFFER_GL1_INV: {
2446 case AMDGPU::SI_RETURN_TO_EPILOG:
2447 case AMDGPU::SI_RETURN:
2448 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2449 case AMDGPU::S_SETPC_B64_return: {
2454 AMDGPU::Waitcnt AllZeroWait =
2455 WCG->getAllZeroWaitcnt(
false);
2456 ScoreBrackets.simplifyVmemInvOnlyWait(AllZeroWait);
2460 case AMDGPU::S_ENDPGM:
2461 case AMDGPU::S_ENDPGM_SAVED: {
2472 !ScoreBrackets.hasPendingEvent(HWEvents::SCRATCH_WRITE_ACCESS);
2475 case AMDGPU::S_SENDMSG:
2476 case AMDGPU::S_SENDMSGHALT: {
2477 if (
ST.hasLegacyGeometry() &&
2492 if (
MI.modifiesRegister(AMDGPU::EXEC, &
TRI)) {
2495 if (ScoreBrackets.hasPendingEvent(HWEvents::EXP_GPR_LOCK) ||
2496 ScoreBrackets.hasPendingEvent(HWEvents::EXP_PARAM_ACCESS) ||
2497 ScoreBrackets.hasPendingEvent(HWEvents::EXP_POS_ACCESS) ||
2498 ScoreBrackets.hasPendingEvent(HWEvents::GDS_GPR_LOCK)) {
2505 if (
TII.isAlwaysGDS(
Opc) && ScoreBrackets.hasPendingGDS())
2513 Wait = AMDGPU::Waitcnt();
2515 const MachineOperand &CallAddrOp =
TII.getCalleeOperand(
MI);
2516 if (CallAddrOp.
isReg()) {
2517 ScoreBrackets.determineWaitForPhysReg(
2520 if (
const auto *RtnAddrOp =
2521 TII.getNamedOperand(
MI, AMDGPU::OpName::dst)) {
2522 ScoreBrackets.determineWaitForPhysReg(
2523 SmemAccessCounter, RtnAddrOp->getReg().asMCReg(),
Wait,
MI);
2526 }
else if (
Opc == AMDGPU::S_BARRIER_WAIT) {
2527 ScoreBrackets.tryClearSCCWriteEvent(&
MI);
2543 for (
const MachineMemOperand *Memop :
MI.memoperands()) {
2544 const Value *Ptr = Memop->getValue();
2545 if (Memop->isStore()) {
2546 if (
auto It = SLoadAddresses.
find(Ptr); It != SLoadAddresses.
end()) {
2547 Wait.add(SmemAccessCounter, 0);
2549 SLoadAddresses.
erase(It);
2552 unsigned AS = Memop->getAddrSpace();
2556 if (
TII.mayWriteLDSThroughDMA(
MI))
2560 VMEMID TID = LDSDMA_BEGIN;
2561 if (Ptr && Memop->getAAInfo()) {
2562 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2563 for (
size_t I = 0,
E = LDSDMAStores.size();
I !=
E; ++
I) {
2564 if (
MI.mayAlias(AA, *LDSDMAStores[
I],
true)) {
2565 size_t Slot =
I + 1;
2566 if (Slot >= NUM_LDSDMA) {
2574 ScoreBrackets.determineWaitForLDSDMA(
2581 if (Memop->isStore()) {
2587 for (
const MachineOperand &
Op :
MI.operands()) {
2592 if (
Op.isTied() &&
Op.isUse() &&
TII.doesNotReadTiedSource(
MI))
2595 MCRegister
Reg =
Op.getReg().asMCReg();
2597 const bool IsVGPR =
TRI.isVectorRegister(
MRI,
Op.getReg());
2604 if (
Op.isImplicit() &&
MI.mayLoadOrStore())
2622 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
2623 ScoreBrackets.hasDifferentVGPRPendingEvents(
2625 ScoreBrackets.hasPointSamplePendingVmemTypes(
MI,
Reg) ||
2626 !
ST.hasVmemWriteVgprInOrder()) {
2633 ScoreBrackets.clearVGPRPendingEvents(
Reg);
2637 ScoreBrackets.hasPendingEvent(HWEvents::EXP_LDS_ACCESS)) {
2642 }
else if (
Op.getReg() == AMDGPU::SCC) {
2645 ScoreBrackets.determineWaitForPhysReg(SmemAccessCounter,
Reg,
Wait,
2649 if (
ST.hasWaitXcnt() &&
ST.isXNACKEnabled() &&
Op.isDef())
2668 if (
Opc == AMDGPU::S_BARRIER && !
ST.hasAutoWaitcntBeforeBarrier() &&
2669 !
ST.hasBackOffBarrier()) {
2670 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
true));
2677 ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS)) {
2682 ScoreBrackets.simplifyWaitcnt(
Wait);
2688 if (
TII.isVALU(
MI,
false)) {
2703 if (
ST.getCLOpts().waitcnt_forcezero && !
MI.isTerminator())
2704 Wait = WCG->getAllZeroWaitcnt(
false);
2708 if (!ForceEmitWaitcnt[
T])
2713 if (FlushFlags.FlushVmCnt) {
2719 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(
AMDGPU::DS_CNT))
2722 if (
ST.getCLOpts().waitcnt_load_forcezero &&
2726 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
2730bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt
Wait,
2732 MachineBasicBlock &
Block,
2733 WaitcntBrackets &ScoreBrackets,
2734 MachineInstr *OldWaitcntInstr) {
2737 if (OldWaitcntInstr)
2741 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
2746 MachineOperand *WaitExp =
TII.getNamedOperand(*It, AMDGPU::OpName::waitexp);
2756 <<
"Update Instr: " << *It);
2759 if (WCG->createNewWaitcnt(
Block, It,
Wait, ScoreBrackets))
2764 ScoreBrackets.applyWaitcnt(
Wait);
2769bool SIInsertWaitcnts::isVmemAccess(
const MachineInstr &
MI)
const {
2770 return (
TII.isFLAT(
MI) &&
TII.mayAccessVMEMThroughFlat(
MI)) ||
2777 MachineBasicBlock *
Block)
const {
2778 auto BlockEnd =
Block->getParent()->end();
2779 auto BlockIter =
Block->getIterator();
2783 if (++BlockIter != BlockEnd) {
2784 It = BlockIter->instr_begin();
2791 if (!It->isMetaInstruction())
2799 return It->getOpcode() == AMDGPU::S_ENDPGM;
2803bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2804 MachineBasicBlock &
Block,
2805 WaitcntBrackets &ScoreBrackets) {
2806 AMDGPU::Waitcnt
Wait;
2807 bool NeedsEndPGMCheck =
false;
2815 NeedsEndPGMCheck =
true;
2818 ScoreBrackets.simplifyWaitcnt(
Wait);
2821 bool Result = generateWaitcnt(
Wait, SuccessorIt,
Block, ScoreBrackets,
2824 if (Result && NeedsEndPGMCheck && isNextENDPGM(SuccessorIt, &
Block)) {
2832void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2833 WaitcntBrackets *ScoreBrackets) {
2837 ScoreBrackets->updateByEvent(
E, Inst);
2839 if (
TII.isDS(Inst) &&
TII.usesLGKM_CNT(Inst)) {
2841 TII.hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
2842 ScoreBrackets->setPendingGDS();
2844 }
else if (
TII.isFLAT(Inst)) {
2845 if (InstEvents.contains(HWEvents::VMEM_GROUP) &&
2846 !
TII.mayAccessVMEMThroughFlat(Inst) &&
2847 !InstEvents.contains(HWEvents::ASYNC_ACCESS))
2848 ScoreBrackets->setPendingXcntOnlyVmem();
2851 TII.mayAccessLDSThroughFlat(Inst, TgSplit) &&
2858 ScoreBrackets->setPendingFlat();
2860 }
else if (Inst.
isCall()) {
2863 ScoreBrackets->applyWaitcnt(WCG->getAllZeroWaitcnt(
false));
2864 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2865 }
else if (
TII.isVINTERP(Inst)) {
2866 int64_t
Imm =
TII.getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2876bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
2877 unsigned OtherScore) {
2878 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
2879 unsigned OtherShifted =
2880 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
2881 Score = std::max(MyShifted, OtherShifted);
2882 return OtherShifted > MyShifted;
2888 bool StrictDom =
false;
2891 LLVM_DEBUG(
dbgs() <<
"Merging async marks of stage " << Stage <<
" ...");
2893 if (StageMarks.empty() && OtherMarks.
empty()) {
2900 auto MaxSize = (unsigned)std::max(StageMarks.size(), OtherMarks.
size());
2901 MaxSize = std::min(MaxSize, MaxAsyncMarks);
2904 if (StageMarks.size() > MaxSize)
2905 StageMarks.erase(StageMarks.begin(),
2906 StageMarks.begin() + (StageMarks.size() - MaxSize));
2912 constexpr CounterValueArray ZeroMark{};
2913 StageMarks.insert(StageMarks.begin(), MaxSize - StageMarks.size(), ZeroMark);
2916 dbgs() <<
"Before merge:\n";
2917 for (
const auto &Mark : StageMarks) {
2921 dbgs() <<
"Other marks:\n";
2922 for (
const auto &Mark : OtherMarks) {
2935 const size_t OtherSize = OtherMarks.size();
2936 const size_t OurSize = StageMarks.size();
2943 "StageMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2946 const CounterValueArray &OtherMark =
2947 Idx <= OtherSize ? OtherMarks[OtherSize -
Idx] : ZeroMark;
2950 mergeScore(MergeInfos[
T], StageMarks[OurSize - Idx][
T], OtherMark[
T]);
2954 dbgs() <<
"After merge:\n";
2955 for (
const auto &Mark : StageMarks) {
2969bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
2970 bool StrictDom =
false;
2974 for (
auto K :
Other.VMem.keys())
2975 VMem.try_emplace(K);
2976 for (
auto K :
Other.SGPRs.keys())
2977 SGPRs.try_emplace(K);
2985 const HWEvents OldEvents = PendingEvents & EventsForT;
2986 const HWEvents OtherEvents =
Other.PendingEvents & EventsForT;
2987 if (!OldEvents.
contains(OtherEvents))
2989 PendingEvents |= OtherEvents;
2992 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
2993 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
2994 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
2995 if (NewUB < ScoreLBs[
T])
2998 MergeInfo &
M = MergeInfos[
T];
2999 M.OldLB = ScoreLBs[
T];
3000 M.OtherLB =
Other.ScoreLBs[
T];
3001 M.MyShift = NewUB - ScoreUBs[
T];
3002 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
3004 ScoreUBs[
T] = NewUB;
3007 StrictDom |= mergeScore(M, LastFlatLoadCnt,
Other.LastFlatLoadCnt);
3010 StrictDom |= mergeScore(M, LastXcntOnlyVmem,
Other.LastXcntOnlyVmem);
3013 StrictDom |= mergeScore(M, LastFlatDsCnt,
Other.LastFlatDsCnt);
3014 StrictDom |= mergeScore(M, LastGDS,
Other.LastGDS);
3018 StrictDom |= mergeScore(M, SCCScore,
Other.SCCScore);
3019 if (
Other.hasPendingEvent(HWEvents::SCC_WRITE)) {
3020 if (!(OldEvents & HWEvents::SCC_WRITE)) {
3021 PendingSCCWrite =
Other.PendingSCCWrite;
3022 }
else if (PendingSCCWrite !=
Other.PendingSCCWrite) {
3023 PendingSCCWrite =
nullptr;
3028 for (
auto &[RegID, Info] : VMem) {
3029 unsigned OtherScore =
Other.getVMemScore(RegID,
T);
3034 bool MyPending =
Info.Scores[
T] >
M.OldLB;
3035 bool OtherPending = OtherScore >
M.OtherLB;
3036 unsigned MyCount = MyPending ? getNumOrderedXDLsAfterWrite(RegID) : 0;
3037 unsigned OtherCount =
3038 OtherPending ?
Other.getNumOrderedXDLsAfterWrite(RegID) : 0;
3039 unsigned NewCount = MyPending && OtherPending
3040 ? std::min(MyCount, OtherCount)
3041 : (MyPending ? MyCount : OtherCount);
3042 StrictDom |= MyPending && NewCount < MyCount;
3043 Info.OrderedXDLCountAtWrite = OrderedXDLCount - NewCount;
3045 StrictDom |= mergeScore(M,
Info.Scores[
T], OtherScore);
3048 if (isSmemCounter(
T)) {
3049 for (
auto &[RegID, Info] : SGPRs) {
3050 auto It =
Other.SGPRs.find(RegID);
3051 unsigned OtherScore = (It !=
Other.SGPRs.end()) ? It->second.get(
T) : 0;
3052 StrictDom |= mergeScore(M,
Info.get(
T), OtherScore);
3057 for (
auto &[TID, Info] : VMem) {
3058 if (
auto It =
Other.VMem.find(TID); It !=
Other.VMem.end()) {
3060 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
3061 StrictDom |= NewVGPRContext !=
Info.VGPRPendingEvents;
3062 Info.VGPRPendingEvents = NewVGPRContext;
3068 StrictDom |= mergeAsyncMarks(S, MergeInfos,
Other.AsyncMarks[SI]);
3071 mergeScore(MergeInfos[
T], AsyncScore[SI][
T],
Other.AsyncScore[SI][
T]);
3074 purgeEmptyTrackingData();
3080 return Opcode == AMDGPU::S_WAITCNT ||
3083 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
3084 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
3085 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
3086 Opcode == AMDGPU::WAIT_ASYNCMARK ||
3090void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &
MBB,
3092 bool ExpertMode)
const {
3096 .
addImm(ExpertMode ? 2 : 0)
3114class VCCZWorkaround {
3115 const WaitcntBrackets &ScoreBrackets;
3116 const GCNSubtarget &
ST;
3117 const SIInstrInfo &
TII;
3118 const SIRegisterInfo &
TRI;
3119 bool VCCZCorruptionBug =
false;
3120 bool VCCZNotUpdatedByPartialWrites =
false;
3123 bool MustRecomputeVCCZ =
true;
3126 VCCZWorkaround(
const WaitcntBrackets &ScoreBrackets,
const GCNSubtarget &ST,
3127 const SIInstrInfo &
TII,
const SIRegisterInfo &
TRI)
3129 VCCZCorruptionBug =
ST.hasReadVCCZBug();
3130 VCCZNotUpdatedByPartialWrites = !
ST.partialVCCWritesUpdateVCCZ();
3137 bool tryRecomputeVCCZ(MachineInstr &
MI) {
3139 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
3149 MustRecomputeVCCZ |= VCCZCorruptionBug &&
TII.isSMRD(
MI);
3155 std::optional<bool> PartiallyWritesToVCCOpt;
3156 auto PartiallyWritesToVCC = [](MachineInstr &
MI) {
3157 return MI.definesRegister(AMDGPU::VCC_LO,
nullptr) ||
3158 MI.definesRegister(AMDGPU::VCC_HI,
nullptr);
3160 if (VCCZNotUpdatedByPartialWrites) {
3161 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(
MI);
3164 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3170 if (!ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS) ||
3171 !VCCZCorruptionBug) {
3173 if (!PartiallyWritesToVCCOpt)
3174 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(
MI);
3175 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3176 MI.definesRegister(AMDGPU::VCC,
nullptr);
3179 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3180 *PartiallyWritesToVCCOpt);
3182 MustRecomputeVCCZ =
false;
3192 TII.get(
ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3195 MustRecomputeVCCZ =
false;
3206 MachineBasicBlock &
Block,
3207 WaitcntBrackets &ScoreBrackets) {
3211 dbgs() <<
"*** Begin Block: ";
3213 ScoreBrackets.dump();
3215 VCCZWorkaround VCCZW(ScoreBrackets, ST,
TII,
TRI);
3218 MachineInstr *OldWaitcntInstr =
nullptr;
3221 ScoreBrackets.verify();
3224 Iter !=
E; ++Iter) {
3225 MachineInstr &Inst = *Iter;
3226 if (isNonWaitcntMetaInst(Inst))
3231 (IsExpertMode && Inst.
getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3232 if (!OldWaitcntInstr)
3233 OldWaitcntInstr = &Inst;
3237 PreheaderFlushFlags FlushFlags;
3238 if (
Block.getFirstTerminator() == Inst)
3239 FlushFlags = isPreheaderToFlush(
Block, ScoreBrackets);
3242 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
3244 OldWaitcntInstr =
nullptr;
3246 if (Inst.
getOpcode() == AMDGPU::ASYNCMARK) {
3255 if (
TII.isSMRD(Inst)) {
3256 for (
const MachineMemOperand *Memop : Inst.
memoperands()) {
3259 if (!Memop->isInvariant()) {
3260 const Value *Ptr = Memop->getValue();
3266 updateEventWaitcntAfter(Inst, &ScoreBrackets);
3270 Modified |= insertForcedWaitAfter(Inst,
Block, ScoreBrackets);
3274 ScoreBrackets.dump();
3279 Modified |= VCCZW.tryRecomputeVCCZ(Inst);
3281 ScoreBrackets.verify();
3286 AMDGPU::Waitcnt
Wait;
3287 if (
Block.getFirstTerminator() ==
Block.end()) {
3288 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(
Block, ScoreBrackets);
3289 if (FlushFlags.FlushVmCnt) {
3297 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(
AMDGPU::DS_CNT))
3306 dbgs() <<
"*** End Block: ";
3308 ScoreBrackets.dump();
3314bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &
Block) {
3315 if (
Block.size() <= 1)
3323 MachineInstr *LastAtomicWithSoftXcnt =
nullptr;
3327 bool IsLDS =
TII.isDS(
MI) ||
3328 (
TII.isFLAT(
MI) &&
TII.mayAccessLDSThroughFlat(
MI, TgSplit));
3329 if (!IsLDS && (
MI.mayLoad() ^
MI.mayStore()))
3330 LastAtomicWithSoftXcnt =
nullptr;
3334 MachineInstr &PrevMI = *
MI.getPrevNode();
3336 if (PrevMI.
getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3339 if (LastAtomicWithSoftXcnt) {
3343 LastAtomicWithSoftXcnt = &
MI;
3351SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &
MBB,
3352 const WaitcntBrackets &ScoreBrackets) {
3360 return PreheaderFlushFlags();
3364 return PreheaderFlushFlags();
3366 if (
Loop->getLoopPreheader() == &
MBB) {
3367 Iterator->second = getPreheaderFlushFlags(
Loop, ScoreBrackets);
3371 return PreheaderFlushFlags();
3374bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
3376 return TII.mayAccessVMEMThroughFlat(
MI);
3380bool SIInsertWaitcnts::isDSRead(
const MachineInstr &
MI)
const {
3386bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(
const MachineInstr &
MI)
const {
3415SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *
ML,
3416 const WaitcntBrackets &Brackets) {
3417 PreheaderFlushFlags
Flags;
3418 bool HasVMemLoad =
false;
3419 bool HasVMemStore =
false;
3420 bool UsesVgprVMEMLoadedOutside =
false;
3421 bool UsesVgprDSReadOutside =
false;
3422 bool VMemInvalidated =
false;
3426 bool TrackSimpleDSOpt =
ST.hasExtendedWaitCounts();
3427 DenseSet<MCRegUnit> VgprUse;
3428 DenseSet<MCRegUnit> VgprDefVMEM;
3429 DenseSet<MCRegUnit> VgprDefDS;
3435 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3436 unsigned DSReadPosition = 0;
3437 bool IsSingleBlock =
ML->getNumBlocks() == 1;
3438 bool TrackDSFlushPoint =
ST.hasExtendedWaitCounts() && IsSingleBlock;
3439 unsigned LastDSFlushPosition = 0;
3441 for (MachineBasicBlock *
MBB :
ML->blocks()) {
3442 for (MachineInstr &
MI : *
MBB) {
3443 if (isVMEMOrFlatVMEM(
MI)) {
3444 HasVMemLoad |=
MI.mayLoad();
3445 HasVMemStore |=
MI.mayStore();
3449 if (mayStoreIncrementingDSCNT(
MI)) {
3452 if (VMemInvalidated)
3454 TrackSimpleDSOpt =
false;
3455 TrackDSFlushPoint =
false;
3457 bool IsDSRead = isDSRead(
MI);
3462 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3463 if (!TrackDSFlushPoint)
3465 if (
auto It = LastDSReadPositionMap.
find(RU);
3466 It != LastDSReadPositionMap.
end()) {
3470 LastDSFlushPosition = std::max(LastDSFlushPosition, It->second);
3474 for (
const MachineOperand &
Op :
MI.all_uses()) {
3475 if (
Op.isDebug() || !
TRI.isVectorRegister(
MRI,
Op.getReg()))
3478 for (MCRegUnit RU :
TRI.regunits(
Op.getReg().asMCReg())) {
3482 VMemInvalidated =
true;
3486 TrackSimpleDSOpt =
false;
3489 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3493 updateDSReadFlushTracking(RU);
3498 VMEMID
ID = toVMEMID(RU);
3502 UsesVgprVMEMLoadedOutside =
true;
3507 UsesVgprDSReadOutside =
true;
3512 if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad()) {
3513 for (
const MachineOperand &
Op :
MI.all_defs()) {
3514 for (MCRegUnit RU :
TRI.regunits(
Op.getReg().asMCReg())) {
3518 VMemInvalidated =
true;
3523 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3534 if (IsDSRead || TrackDSFlushPoint) {
3535 for (
const MachineOperand &
Op :
MI.all_defs()) {
3536 if (!
TRI.isVectorRegister(
MRI,
Op.getReg()))
3538 for (MCRegUnit RU :
TRI.regunits(
Op.getReg().asMCReg())) {
3541 updateDSReadFlushTracking(RU);
3544 if (TrackDSFlushPoint)
3545 LastDSReadPositionMap[RU] = DSReadPosition;
3554 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3555 ((!
ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3556 (HasVMemLoad &&
ST.hasVmemWriteVgprInOrder())))
3557 Flags.FlushVmCnt =
true;
3563 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3566 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3567 bool DSFlushPointPrefetch =
3568 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3570 if (SimpleDSOpt || DSFlushPointPrefetch)
3571 Flags.FlushDsCnt =
true;
3576bool SIInsertWaitcntsLegacy::runOnMachineFunction(
MachineFunction &MF) {
3577 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3579 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3581 if (
auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3582 AA = &AAR->getAAResults();
3584 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3596 if (!SIInsertWaitcnts(MLI, PDT,
AA, MF).
run())
3601 .preserve<AAManager>();
3604bool SIInsertWaitcnts::run() {
3612 if (ST.hasExtendedWaitCounts()) {
3617 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(MF, MaxCounter, Limits,
3622 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3626 SmemAccessCounter = getCounterFromEvent(HWEvents::SMEM_ACCESS);
3630 MachineBasicBlock &EntryBB = MF.
front();
3641 while (
I != EntryBB.
end() &&
I->isMetaInstruction())
3644 if (
ST.hasExtendedWaitCounts()) {
3653 if (!
ST.hasImageInsts() &&
3662 TII.get(instrsForExtendedCounterTypes[CT]))
3675 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
this);
3676 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3677 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3684 for (
auto *
MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3687 std::unique_ptr<WaitcntBrackets> Brackets;
3692 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
3694 MachineBasicBlock *
MBB = BII->first;
3695 BlockInfo &BI = BII->second;
3701 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
3703 *Brackets = *BI.Incoming;
3706 Brackets = std::make_unique<WaitcntBrackets>(
this);
3711 Brackets->~WaitcntBrackets();
3712 new (Brackets.get()) WaitcntBrackets(
this);
3716 if (
ST.hasWaitXcnt() &&
ST.isXNACKEnabled())
3718 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
3721 if (Brackets->hasPendingEvent()) {
3722 BlockInfo *MoveBracketsToSucc =
nullptr;
3724 auto *SuccBII = BlockInfos.
find(Succ);
3725 BlockInfo &SuccBI = SuccBII->second;
3726 if (!SuccBI.Incoming) {
3727 SuccBI.Dirty =
true;
3728 if (SuccBII <= BII) {
3732 if (!MoveBracketsToSucc) {
3733 MoveBracketsToSucc = &SuccBI;
3735 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
3739 dbgs() <<
"Try to merge ";
3745 if (SuccBI.Incoming->merge(*Brackets)) {
3746 SuccBI.Dirty =
true;
3747 if (SuccBII <= BII) {
3754 if (MoveBracketsToSucc)
3755 MoveBracketsToSucc->Incoming = std::move(Brackets);
3760 Modified |= WCG->eraseDeferredWaitcnts();
3762 if (
ST.hasScalarStores()) {
3763 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3764 bool HaveScalarStores =
false;
3766 for (MachineBasicBlock &
MBB : MF) {
3767 for (MachineInstr &
MI :
MBB) {
3768 if (!HaveScalarStores &&
TII.isScalarStore(
MI))
3769 HaveScalarStores =
true;
3771 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
3772 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3777 if (HaveScalarStores) {
3786 for (MachineBasicBlock *
MBB : EndPgmBlocks) {
3787 bool SeenDCacheWB =
false;
3791 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
3792 SeenDCacheWB =
true;
3793 else if (
TII.isScalarStore(*
I))
3794 SeenDCacheWB =
false;
3797 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
3798 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3814 while (
I != EntryBB.
end() &&
I->isMetaInstruction())
3816 setSchedulingMode(EntryBB,
I,
true);
3818 for (MachineInstr *
MI : CallInsts) {
3820 setSchedulingMode(
MBB,
MI,
false);
3821 setSchedulingMode(
MBB, std::next(
MI->getIterator()),
true);
3824 for (MachineInstr *
MI : ReturnInsts)
3825 setSchedulingMode(*
MI->getParent(),
MI,
false);
3836 for (
auto [
MI,
_] : EndPgmInsts) {
3838 TII.get(AMDGPU::S_ALLOC_VGPR))
3842 }
else if (!WCG->isOptNone() &&
3843 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3844 (MF.getFrameInfo().hasCalls() ||
3845 ST.getOccupancyWithNumVGPRs(
3846 TRI.getNumUsedPhysRegs(
MRI, AMDGPU::VGPR_32RegClass),
3847 false) <
ST.getMaxWavesPerEU())) {
3848 for (
auto [
MI, Flag] : EndPgmInsts) {
3850 if (
ST.requiresNopBeforeDeallocVGPRs()) {
3852 TII.get(AMDGPU::S_NOP))
3856 TII.get(AMDGPU::S_SENDMSG))
3866 if (
ST.hasWaitXcnt() &&
ST.isXNACKEnabled()) {
3873 TII.get(AMDGPU::S_SETREG_IMM32_B32))
3879 if (
ST.hasRequiresInitialUnclausedVmem()) {
3885 AMDGPU::SGPR64_SGPR65)
3889 TII.get(AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3890 .
addReg(AMDGPU::SGPR64_SGPR65)
3891 .
addReg(AMDGPU::VGPR0, RegState::Undef)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Shared AMDGPU asyncmark stage definitions.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static bool isReg(const MCInst &MI, unsigned OpNo)
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
MachineRegisterInfo & MRI
static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
AMDGPU::HWEvents HWEvents
static Function * getFunction(FunctionType *Ty, const Twine &Name, Module *M)
static const uint32_t IV[8]
A manager for alias analyses.
bool isEntryFunction() const
Bit mask of hardware events.
constexpr unsigned size() const
constexpr bool contains(HWEvents Other) const
constexpr bool any() const
unsigned get(InstCounterType T) const
void set(InstCounterType T, unsigned Val)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
Represents analyses that only rely on functions' control flow.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
FunctionPass class - This class is used to implement most global optimizations.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
LLVM_ABI const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
LLVM_ABI void printName(raw_ostream &os, unsigned printNameFlags=PrintNameIr, ModuleSlotTracker *moduleSlotTracker=nullptr) const
Print the basic block's name as:
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Representation of each machine instruction.
mop_range defs()
Returns all explicit operands that are register definitions.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
filtered_mop_range all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
bool isCall(QueryType Type=AnyInBundle) const
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
LLVM_ABI void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
filtered_mop_range all_uses()
Returns an iterator range over all operands that are (explicit or implicit) register uses.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Analysis pass that exposes the MachineLoopInfo for a machine function.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
iterator find(const KeyT &Key)
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
static bool isCBranchVCCZRead(const MachineInstr &MI)
static bool isDS(const MachineInstr &MI)
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isXcntDrain(const MachineInstr &MI)
True if MI implicitly drains XCNT.
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isSBarrierSCCWrite(unsigned Opcode)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
static bool isLDSDMA(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool isDynamicVGPREnabled() const
void push_back(const T &Elt)
Target - Wrapper for Target specific information.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst)
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc)
unsigned decodeFieldVaVdst(unsigned Encoded)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
unsigned decodeFieldVmVsrc(unsigned Encoded)
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
constexpr AsyncStages asyncStagesFromMaskOperand(int64_t Mask)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
iota_range< InstCounterType > inst_counter_types(InstCounterType MaxCounter)
unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool getHasMatrixScale(unsigned Opc)
AsyncStages
Bit mask of the async stages tracked by the asyncmark / wait_asyncmark intrinsics.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded)
unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool isTgSplitEnabled(const Function &F)
bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F)
Return whether expert scheduling mode is available and enabled for F.
HWEvents getSimplifiedVMEMEventsFor(const MachineInstr &Inst, const SIInstrInfo &TII)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
auto asyncStages(AsyncStages Mask)
unsigned getAsyncStageIndex(AsyncStages Stage)
std::optional< AMDGPU::InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
constexpr unsigned NumAsyncStages
The number of stages, which is also the number of bits in a stage mask.
HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST, bool IsExpertMode, bool TgSplit)
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
unsigned encodeStorecntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
constexpr bool any(E Val)
constexpr bool isMaybeAtomic(const T &...O)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr auto seq_inclusive(T Begin, T End)
Iterate over an integral type from Begin to End inclusive.
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
auto accumulate(R &&Range, E &&Init)
Wrapper for std::accumulate.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
LLVM_ABI Printable printRegUnit(MCRegUnit Unit, const TargetRegisterInfo *TRI)
Create Printable object to print register units on a raw_ostream.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
char & SIInsertWaitcntsID
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
@ Increment
Incrementally increasing token ID.
FunctionPass * createSIInsertWaitcntsPass()
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
MCRegisterClass TargetRegisterClass
static constexpr ValueType Default
static constexpr uint64_t encode(Fields... Values)
Represents the hardware counter limits for different wait count types.
Instruction set architecture version.