42#define DEBUG_TYPE "si-insert-waitcnts"
45 "Force emit s_waitcnt expcnt(0) instrs");
47 "Force emit s_waitcnt lgkmcnt(0) instrs");
49 "Force emit s_waitcnt vmcnt(0) instrs");
52 "amdgpu-waitcnt-forcezero",
53 cl::desc(
"Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
67 SAMPLE_CNT = NUM_NORMAL_INST_CNTS,
70 NUM_EXTENDED_INST_CNTS,
71 NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
85auto inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS) {
86 return enum_seq(LOAD_CNT, MaxCounter);
89using RegInterval = std::pair<int, int>;
91struct HardwareLimits {
96 unsigned SamplecntMax;
101struct RegisterEncoding {
111 VMEM_SAMPLER_READ_ACCESS,
112 VMEM_BVH_READ_ACCESS,
114 SCRATCH_WRITE_ACCESS,
134enum RegisterMapping {
135 SQ_MAX_PGM_VGPRS = 512,
137 SQ_MAX_PGM_SGPRS = 256,
145 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS,
166static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
177static bool isNormalMode(InstCounterType MaxCounter) {
178 return MaxCounter == NUM_NORMAL_INST_CNTS;
183 assert(updateVMCntOnly(Inst));
186 return VMEM_NOSAMPLER;
193 return BaseInfo->
BVH ? VMEM_BVH
207 return Wait.StoreCnt;
209 return Wait.SampleCnt;
220 unsigned &WC = getCounterRef(
Wait,
T);
221 WC = std::min(WC, Count);
225 getCounterRef(
Wait,
T) = ~0
u;
229 return getCounterRef(
Wait,
T);
233InstCounterType eventCounter(
const unsigned *masks, WaitEventType E) {
234 for (
auto T : inst_counter_types()) {
235 if (masks[
T] & (1 << E))
249class WaitcntBrackets {
251 WaitcntBrackets(
const GCNSubtarget *SubTarget, InstCounterType MaxCounter,
252 HardwareLimits Limits, RegisterEncoding Encoding,
253 const unsigned *WaitEventMaskForInst,
254 InstCounterType SmemAccessCounter)
255 :
ST(SubTarget), MaxCounter(MaxCounter), Limits(Limits),
256 Encoding(Encoding), WaitEventMaskForInst(WaitEventMaskForInst),
257 SmemAccessCounter(SmemAccessCounter) {}
259 unsigned getWaitCountMax(InstCounterType
T)
const {
262 return Limits.LoadcntMax;
264 return Limits.DscntMax;
266 return Limits.ExpcntMax;
268 return Limits.StorecntMax;
270 return Limits.SamplecntMax;
272 return Limits.BvhcntMax;
274 return Limits.KmcntMax;
281 unsigned getScoreLB(InstCounterType
T)
const {
286 unsigned getScoreUB(InstCounterType
T)
const {
291 unsigned getScoreRange(InstCounterType
T)
const {
292 return getScoreUB(
T) - getScoreLB(
T);
295 unsigned getRegScore(
int GprNo, InstCounterType
T)
const {
296 if (GprNo < NUM_ALL_VGPRS) {
297 return VgprScores[
T][GprNo];
299 assert(
T == SmemAccessCounter);
300 return SgprScores[GprNo - NUM_ALL_VGPRS];
309 bool counterOutOfOrder(InstCounterType
T)
const;
311 void simplifyWaitcnt(InstCounterType
T,
unsigned &Count)
const;
314 void applyWaitcnt(InstCounterType
T,
unsigned Count);
319 unsigned hasPendingEvent()
const {
return PendingEvents; }
320 unsigned hasPendingEvent(WaitEventType E)
const {
321 return PendingEvents & (1 << E);
323 unsigned hasPendingEvent(InstCounterType
T)
const {
324 unsigned HasPending = PendingEvents & WaitEventMaskForInst[
T];
325 assert((HasPending != 0) == (getScoreRange(
T) != 0));
329 bool hasMixedPendingEvents(InstCounterType
T)
const {
330 unsigned Events = hasPendingEvent(
T);
332 return Events & (Events - 1);
335 bool hasPendingFlat()
const {
336 return ((LastFlat[DS_CNT] > ScoreLBs[DS_CNT] &&
337 LastFlat[DS_CNT] <= ScoreUBs[DS_CNT]) ||
338 (LastFlat[LOAD_CNT] > ScoreLBs[LOAD_CNT] &&
339 LastFlat[LOAD_CNT] <= ScoreUBs[LOAD_CNT]));
342 void setPendingFlat() {
343 LastFlat[LOAD_CNT] = ScoreUBs[LOAD_CNT];
344 LastFlat[DS_CNT] = ScoreUBs[DS_CNT];
349 bool hasOtherPendingVmemTypes(
int GprNo, VmemType V)
const {
350 assert(GprNo < NUM_ALL_VGPRS);
351 return VgprVmemTypes[GprNo] & ~(1 <<
V);
354 void clearVgprVmemTypes(
int GprNo) {
355 assert(GprNo < NUM_ALL_VGPRS);
356 VgprVmemTypes[GprNo] = 0;
359 void setStateOnFunctionEntryOrReturn() {
360 setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getWaitCountMax(STORE_CNT));
361 PendingEvents |= WaitEventMaskForInst[STORE_CNT];
378 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
379 unsigned OtherScore);
381 void setScoreLB(InstCounterType
T,
unsigned Val) {
386 void setScoreUB(InstCounterType
T,
unsigned Val) {
393 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
397 void setRegScore(
int GprNo, InstCounterType
T,
unsigned Val) {
398 if (GprNo < NUM_ALL_VGPRS) {
399 VgprUB = std::max(VgprUB, GprNo);
400 VgprScores[
T][GprNo] = Val;
402 assert(
T == SmemAccessCounter);
403 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
404 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
410 unsigned OpNo,
unsigned Val);
413 InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
414 HardwareLimits Limits = {};
415 RegisterEncoding Encoding = {};
416 const unsigned *WaitEventMaskForInst;
417 InstCounterType SmemAccessCounter;
418 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
419 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
420 unsigned PendingEvents = 0;
422 unsigned LastFlat[NUM_INST_CNTS] = {0};
427 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
430 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
433 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
445class WaitcntGenerator {
450 InstCounterType MaxCounter;
454 WaitcntGenerator() {}
455 WaitcntGenerator(
const MachineFunction &MF, InstCounterType MaxCounter)
463 bool isOptNone()
const {
return OptNone; }
477 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
492 virtual const unsigned *getWaitEventMask()
const = 0;
496 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const = 0;
498 virtual ~WaitcntGenerator() =
default;
501 static constexpr unsigned
502 eventMask(std::initializer_list<WaitEventType> Events) {
504 for (
auto &E : Events)
511class WaitcntGeneratorPreGFX12 :
public WaitcntGenerator {
513 WaitcntGeneratorPreGFX12() {}
515 : WaitcntGenerator(MF, NUM_NORMAL_INST_CNTS) {}
518 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
526 const unsigned *getWaitEventMask()
const override {
529 static const unsigned WaitEventMaskForInstPreGFX12[NUM_INST_CNTS] = {
530 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS,
531 VMEM_BVH_READ_ACCESS}),
532 eventMask({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
533 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
534 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
535 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
540 return WaitEventMaskForInstPreGFX12;
543 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
546class WaitcntGeneratorGFX12Plus :
public WaitcntGenerator {
548 WaitcntGeneratorGFX12Plus() {}
550 InstCounterType MaxCounter)
551 : WaitcntGenerator(MF, MaxCounter) {}
554 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
562 const unsigned *getWaitEventMask()
const override {
565 static const unsigned WaitEventMaskForInstGFX12Plus[NUM_INST_CNTS] = {
566 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS}),
567 eventMask({LDS_ACCESS, GDS_ACCESS}),
568 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
569 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
570 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
571 eventMask({VMEM_SAMPLER_READ_ACCESS}),
572 eventMask({VMEM_BVH_READ_ACCESS}),
573 eventMask({SMEM_ACCESS, SQ_MESSAGE})};
575 return WaitEventMaskForInstGFX12Plus;
578 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
595 std::unique_ptr<WaitcntBrackets>
Incoming;
599 InstCounterType SmemAccessCounter;
605 bool ForceEmitZeroWaitcnts;
606 bool ForceEmitWaitcnt[NUM_INST_CNTS];
611 WaitcntGeneratorPreGFX12 WCGPreGFX12;
612 WaitcntGeneratorGFX12Plus WCGGFX12Plus;
614 WaitcntGenerator *WCG =
nullptr;
620 InstCounterType MaxCounter = NUM_NORMAL_INST_CNTS;
626 (void)ForceExpCounter;
627 (void)ForceLgkmCounter;
628 (void)ForceVMCounter;
631 bool shouldFlushVmCnt(
MachineLoop *
ML, WaitcntBrackets &Brackets);
633 WaitcntBrackets &ScoreBrackets);
638 return "SI insert wait instructions";
650 bool isForceEmitWaitcnt()
const {
651 for (
auto T : inst_counter_types())
652 if (ForceEmitWaitcnt[
T])
657 void setForceEmitWaitcnt() {
663 ForceEmitWaitcnt[
EXP_CNT] =
true;
665 ForceEmitWaitcnt[
EXP_CNT] =
false;
670 ForceEmitWaitcnt[DS_CNT] =
true;
671 ForceEmitWaitcnt[KM_CNT] =
true;
673 ForceEmitWaitcnt[DS_CNT] =
false;
674 ForceEmitWaitcnt[KM_CNT] =
false;
679 ForceEmitWaitcnt[LOAD_CNT] =
true;
680 ForceEmitWaitcnt[SAMPLE_CNT] =
true;
681 ForceEmitWaitcnt[BVH_CNT] =
true;
683 ForceEmitWaitcnt[LOAD_CNT] =
false;
684 ForceEmitWaitcnt[SAMPLE_CNT] =
false;
685 ForceEmitWaitcnt[BVH_CNT] =
false;
692 WaitEventType getVmemWaitEventType(
const MachineInstr &Inst)
const {
694 static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
695 VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
706 return SCRATCH_WRITE_ACCESS;
707 return VMEM_WRITE_ACCESS;
710 return VMEM_READ_ACCESS;
711 return VmemReadMapping[getVmemType(Inst)];
718 WaitcntBrackets &ScoreBrackets,
726 WaitcntBrackets *ScoreBrackets);
728 WaitcntBrackets &ScoreBrackets);
733RegInterval WaitcntBrackets::getRegInterval(
const MachineInstr *
MI,
736 unsigned OpNo)
const {
738 if (!
TRI->isInAllocatableClass(
Op.getReg()))
750 if (
TRI->isVectorRegister(*
MRI,
Op.getReg())) {
751 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
754 Result.first += AGPR_OFFSET;
756 }
else if (
TRI->isSGPRReg(*
MRI,
Op.getReg())) {
757 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
758 Result.first =
Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
760 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
768 unsigned Size =
TRI->getRegSizeInBits(*RC);
780 assert(
TRI->isVectorRegister(*
MRI,
MI->getOperand(OpNo).getReg()));
782 setRegScore(RegNo, EXP_CNT, Val);
790 InstCounterType
T = eventCounter(WaitEventMaskForInst, E);
792 unsigned UB = getScoreUB(
T);
793 unsigned CurrScore = UB + 1;
799 PendingEvents |= 1 << E;
800 setScoreUB(
T, CurrScore);
810 if (AddrOpIdx != -1) {
811 setExpScore(&Inst,
TII,
TRI,
MRI, AddrOpIdx, CurrScore);
824 AMDGPU::OpName::data1),
829 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
830 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
833 if (
Op.isReg() && !
Op.isDef() &&
834 TRI->isVectorRegister(*
MRI,
Op.getReg())) {
835 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
839 }
else if (
TII->isFLAT(Inst)) {
851 }
else if (
TII->isMIMG(Inst)) {
853 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
860 }
else if (
TII->isMTBUF(Inst)) {
862 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
864 }
else if (
TII->isMUBUF(Inst)) {
866 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
873 }
else if (
TII->isLDSDIR(Inst)) {
880 if (
TII->isEXP(Inst)) {
899 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
904 }
else if (Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORD ||
905 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 ||
906 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) {
912 setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore);
919 if (!
Op.isReg() || !
Op.isDef())
922 if (
T == LOAD_CNT ||
T == SAMPLE_CNT ||
T == BVH_CNT) {
923 if (
Interval.first >= NUM_ALL_VGPRS)
925 if (updateVMCntOnly(Inst)) {
930 VmemType
V = getVmemType(Inst);
932 VgprVmemTypes[RegNo] |= 1 <<
V;
936 setRegScore(RegNo,
T, CurrScore);
940 (
TII->isDS(Inst) ||
TII->mayWriteLDSThroughDMA(Inst))) {
945 if (!
MemOp->isStore() ||
950 auto AAI =
MemOp->getAAInfo();
958 if (!AAI || !AAI.Scope)
960 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E && !Slot; ++
I) {
961 for (
const auto *
MemOp : LDSDMAStores[
I]->memoperands()) {
962 if (
MemOp->isStore() && AAI ==
MemOp->getAAInfo()) {
968 if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
970 LDSDMAStores.push_back(&Inst);
971 Slot = LDSDMAStores.size();
974 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot,
T, CurrScore);
976 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS,
T, CurrScore);
983 for (
auto T : inst_counter_types(MaxCounter)) {
984 unsigned SR = getScoreRange(
T);
988 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"LOAD" :
"VM") <<
"_CNT("
992 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"DS" :
"LGKM") <<
"_CNT("
996 OS <<
" EXP_CNT(" << SR <<
"): ";
999 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"STORE" :
"VS") <<
"_CNT("
1003 OS <<
" SAMPLE_CNT(" << SR <<
"): ";
1006 OS <<
" BVH_CNT(" << SR <<
"): ";
1009 OS <<
" KM_CNT(" << SR <<
"): ";
1012 OS <<
" UNKNOWN(" << SR <<
"): ";
1018 unsigned LB = getScoreLB(
T);
1020 for (
int J = 0; J <= VgprUB; J++) {
1021 unsigned RegScore = getRegScore(J,
T);
1024 unsigned RelScore = RegScore - LB - 1;
1025 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
1026 OS << RelScore <<
":v" << J <<
" ";
1028 OS << RelScore <<
":ds ";
1032 if (
T == SmemAccessCounter) {
1033 for (
int J = 0; J <= SgprUB; J++) {
1034 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS,
T);
1037 unsigned RelScore = RegScore - LB - 1;
1038 OS << RelScore <<
":s" << J <<
" ";
1050 simplifyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1051 simplifyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1052 simplifyWaitcnt(DS_CNT,
Wait.DsCnt);
1053 simplifyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1054 simplifyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1055 simplifyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1056 simplifyWaitcnt(KM_CNT,
Wait.KmCnt);
1059void WaitcntBrackets::simplifyWaitcnt(InstCounterType
T,
1060 unsigned &Count)
const {
1064 if (Count >= getScoreRange(
T))
1068void WaitcntBrackets::determineWait(InstCounterType
T,
int RegNo,
1070 unsigned ScoreToWait = getRegScore(RegNo,
T);
1074 const unsigned LB = getScoreLB(
T);
1075 const unsigned UB = getScoreUB(
T);
1076 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1077 if ((
T == LOAD_CNT ||
T == DS_CNT) && hasPendingFlat() &&
1078 !
ST->hasFlatLgkmVMemCountInOrder()) {
1082 addWait(
Wait,
T, 0);
1083 }
else if (counterOutOfOrder(
T)) {
1087 addWait(
Wait,
T, 0);
1091 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(
T) - 1);
1092 addWait(
Wait,
T, NeededWait);
1098 applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1099 applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1100 applyWaitcnt(DS_CNT,
Wait.DsCnt);
1101 applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1102 applyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1103 applyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1104 applyWaitcnt(KM_CNT,
Wait.KmCnt);
1107void WaitcntBrackets::applyWaitcnt(InstCounterType
T,
unsigned Count) {
1108 const unsigned UB = getScoreUB(
T);
1112 if (counterOutOfOrder(
T))
1114 setScoreLB(
T, std::max(getScoreLB(
T), UB - Count));
1117 PendingEvents &= ~WaitEventMaskForInst[
T];
1123bool WaitcntBrackets::counterOutOfOrder(InstCounterType
T)
const {
1125 if (
T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
1127 return hasMixedPendingEvents(
T);
1137char SIInsertWaitcnts::
ID = 0;
1142 return new SIInsertWaitcnts();
1152 if (NewEnc == MO.
getImm())
1163 case AMDGPU::S_WAIT_LOADCNT:
1165 case AMDGPU::S_WAIT_EXPCNT:
1167 case AMDGPU::S_WAIT_STORECNT:
1169 case AMDGPU::S_WAIT_SAMPLECNT:
1171 case AMDGPU::S_WAIT_BVHCNT:
1173 case AMDGPU::S_WAIT_DSCNT:
1175 case AMDGPU::S_WAIT_KMCNT:
1182bool WaitcntGenerator::promoteSoftWaitCnt(
MachineInstr *Waitcnt)
const {
1196bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1197 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1200 assert(isNormalMode(MaxCounter));
1208 if (II.isMetaInstruction())
1212 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1216 if (Opcode == AMDGPU::S_WAITCNT) {
1217 unsigned IEnc = II.getOperand(0).getImm();
1220 ScoreBrackets.simplifyWaitcnt(OldWait);
1224 if (WaitcntInstr || (!
Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1225 II.eraseFromParent();
1230 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1231 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1234 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1236 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt);
1237 Wait.StoreCnt = std::min(
Wait.StoreCnt, OldVSCnt);
1239 if (WaitcntVsCntInstr || (!
Wait.hasWaitStoreCnt() && TrySimplify)) {
1240 II.eraseFromParent();
1243 WaitcntVsCntInstr = &II;
1250 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1252 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1253 ScoreBrackets.applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1254 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1261 <<
"applyPreexistingWaitcnt\n"
1262 <<
"New Instr at block end: " << *WaitcntInstr <<
'\n'
1263 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1264 <<
"Old Instr: " << *It
1265 <<
"New Instr: " << *WaitcntInstr <<
'\n');
1268 if (WaitcntVsCntInstr) {
1270 AMDGPU::OpName::simm16,
Wait.StoreCnt);
1271 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1273 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1274 Wait.StoreCnt = ~0
u;
1277 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1278 <<
"New Instr at block end: " << *WaitcntVsCntInstr
1280 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1281 <<
"Old Instr: " << *It
1282 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
1290bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1294 assert(isNormalMode(MaxCounter));
1301 if (
Wait.hasWaitExceptStoreCnt()) {
1303 [[maybe_unused]]
auto SWaitInst =
1308 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1309 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1312 if (
Wait.hasWaitStoreCnt()) {
1315 [[maybe_unused]]
auto SWaitInst =
1322 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1323 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1330WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1335WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1343bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1344 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1347 assert(!isNormalMode(MaxCounter));
1356 if (II.isMetaInstruction())
1365 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1367 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1369 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1372 ScoreBrackets.simplifyWaitcnt(OldWait);
1374 UpdatableInstr = &CombinedLoadDsCntInstr;
1375 }
else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1377 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1380 ScoreBrackets.simplifyWaitcnt(OldWait);
1382 UpdatableInstr = &CombinedStoreDsCntInstr;
1387 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1389 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt);
1390 addWait(
Wait, CT.value(), OldCnt);
1391 UpdatableInstr = &WaitInstrs[CT.value()];
1395 if (!*UpdatableInstr) {
1396 *UpdatableInstr = &II;
1403 if (CombinedLoadDsCntInstr) {
1411 if (
Wait.LoadCnt != ~0u &&
Wait.DsCnt != ~0u) {
1414 AMDGPU::OpName::simm16, NewEnc);
1415 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
1416 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1417 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1422 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1423 <<
"New Instr at block end: "
1424 << *CombinedLoadDsCntInstr <<
'\n'
1425 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1426 <<
"Old Instr: " << *It <<
"New Instr: "
1427 << *CombinedLoadDsCntInstr <<
'\n');
1434 if (CombinedStoreDsCntInstr) {
1436 if (
Wait.StoreCnt != ~0u &&
Wait.DsCnt != ~0u) {
1439 AMDGPU::OpName::simm16, NewEnc);
1440 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
1441 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1442 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1443 Wait.StoreCnt = ~0
u;
1447 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1448 <<
"New Instr at block end: "
1449 << *CombinedStoreDsCntInstr <<
'\n'
1450 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1451 <<
"Old Instr: " << *It <<
"New Instr: "
1452 << *CombinedStoreDsCntInstr <<
'\n');
1465 if (
Wait.DsCnt != ~0u) {
1474 if (
Wait.LoadCnt != ~0u) {
1475 WaitsToErase.
push_back(&WaitInstrs[LOAD_CNT]);
1476 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1477 }
else if (
Wait.StoreCnt != ~0u) {
1478 WaitsToErase.
push_back(&WaitInstrs[STORE_CNT]);
1479 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1486 (*WI)->eraseFromParent();
1492 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1493 if (!WaitInstrs[CT])
1496 unsigned NewCnt = getWait(
Wait, CT);
1497 if (NewCnt != ~0u) {
1499 AMDGPU::OpName::simm16, NewCnt);
1500 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
1502 ScoreBrackets.applyWaitcnt(CT, NewCnt);
1503 setNoWait(
Wait, CT);
1506 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1507 <<
"New Instr at block end: " << *WaitInstrs[CT]
1509 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1510 <<
"Old Instr: " << *It
1511 <<
"New Instr: " << *WaitInstrs[CT] <<
'\n');
1522bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
1526 assert(!isNormalMode(MaxCounter));
1532 if (
Wait.DsCnt != ~0u) {
1535 if (
Wait.LoadCnt != ~0u) {
1543 }
else if (
Wait.StoreCnt != ~0u) {
1550 Wait.StoreCnt = ~0
u;
1558 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1559 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1566 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1567 unsigned Count = getWait(
Wait, CT);
1571 [[maybe_unused]]
auto SWaitInst =
1578 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1579 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1586 unsigned Opc =
MI.getOpcode();
1587 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
1588 !
MI.getOperand(1).isUndef();
1618bool SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &
MI,
1619 WaitcntBrackets &ScoreBrackets,
1622 setForceEmitWaitcnt();
1624 if (
MI.isMetaInstruction())
1633 if (
MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1634 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1635 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1636 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1637 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1644 if (
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1645 MI.getOpcode() == AMDGPU::SI_RETURN ||
1646 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1648 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
false));
1656 else if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1657 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1659 ScoreBrackets.getScoreRange(STORE_CNT) != 0 &&
1660 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1664 else if ((
MI.getOpcode() == AMDGPU::S_SENDMSG ||
1665 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1666 ST->hasLegacyGeometry() &&
1672 else if (
MI.getOpcode() == SC_FENCE) {
1673 const unsigned int group_size =
1674 context->shader_info->GetMaxThreadGroupSize();
1676 const bool group_is_multi_wave =
1677 (group_size == 0 || group_size > target_info->GetWaveFrontSize());
1678 const bool fence_is_global = !((SCInstInternalMisc*)Inst)->IsGroupFence();
1680 for (
unsigned int i = 0; i < Inst->NumSrcOperands(); i++) {
1681 SCRegType src_type = Inst->GetSrcType(i);
1684 if (group_is_multi_wave ||
1685 context->OptFlagIsOn(OPT_R1100_LDSMEM_FENCE_CHICKEN_BIT)) {
1686 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1687 ScoreBrackets->getScoreUB(DS_CNT));
1689 if (target_info->HasBufferLoadToLDS()) {
1690 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1691 ScoreBrackets->getScoreUB(LOAD_CNT));
1697 if (group_is_multi_wave || fence_is_global) {
1698 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1699 ScoreBrackets->getScoreUB(EXP_CNT));
1700 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1701 ScoreBrackets->getScoreUB(DS_CNT));
1709 if (group_is_multi_wave || fence_is_global) {
1710 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1711 ScoreBrackets->getScoreUB(EXP_CNT));
1712 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1713 ScoreBrackets->getScoreUB(LOAD_CNT));
1730 if (
MI.modifiesRegister(AMDGPU::EXEC,
TRI)) {
1733 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1734 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1735 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1736 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1750 if (
MI.getOperand(CallAddrOpIdx).isReg()) {
1751 RegInterval CallAddrOpInterval =
1752 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, CallAddrOpIdx);
1754 for (
int RegNo = CallAddrOpInterval.first;
1755 RegNo < CallAddrOpInterval.second; ++RegNo)
1756 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1760 if (RtnAddrOpIdx != -1) {
1761 RegInterval RtnAddrOpInterval =
1762 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, RtnAddrOpIdx);
1764 for (
int RegNo = RtnAddrOpInterval.first;
1765 RegNo < RtnAddrOpInterval.second; ++RegNo)
1766 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1785 const Value *
Ptr = Memop->getValue();
1786 if (Memop->isStore() && SLoadAddresses.
count(
Ptr)) {
1787 addWait(
Wait, SmemAccessCounter, 0);
1791 unsigned AS = Memop->getAddrSpace();
1795 if (
TII->mayWriteLDSThroughDMA(
MI))
1799 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1800 bool FoundAliasingStore =
false;
1807 if (
Ptr && Memop->getAAInfo() && Memop->getAAInfo().Scope) {
1808 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
1809 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E; ++
I) {
1810 if (
MI.mayAlias(AA, *LDSDMAStores[
I],
true)) {
1811 FoundAliasingStore =
true;
1812 ScoreBrackets.determineWait(LOAD_CNT, RegNo +
I + 1,
Wait);
1816 if (!FoundAliasingStore)
1817 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1818 if (Memop->isStore()) {
1819 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1824 for (
unsigned I = 0, E =
MI.getNumOperands();
I != E; ++
I) {
1830 if (
Op.isTied() &&
Op.isUse() &&
TII->doesNotReadTiedSource(
MI))
1835 const bool IsVGPR =
TRI->isVectorRegister(*
MRI,
Op.getReg());
1842 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
1843 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1845 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1846 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo,
Wait);
1847 ScoreBrackets.determineWait(BVH_CNT, RegNo,
Wait);
1848 ScoreBrackets.clearVgprVmemTypes(RegNo);
1850 if (
Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1851 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1853 ScoreBrackets.determineWait(DS_CNT, RegNo,
Wait);
1855 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1866 if (
TII->isBarrierStart(
MI.getOpcode()) &&
1867 !
ST->hasAutoWaitcntBeforeBarrier() && !
ST->supportsBackOffBarrier()) {
1868 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
true));
1875 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1881 ScoreBrackets.simplifyWaitcnt(
Wait);
1883 if (ForceEmitZeroWaitcnts)
1884 Wait = WCG->getAllZeroWaitcnt(
false);
1886 if (ForceEmitWaitcnt[LOAD_CNT])
1888 if (ForceEmitWaitcnt[EXP_CNT])
1890 if (ForceEmitWaitcnt[DS_CNT])
1892 if (ForceEmitWaitcnt[SAMPLE_CNT])
1894 if (ForceEmitWaitcnt[BVH_CNT])
1896 if (ForceEmitWaitcnt[KM_CNT])
1900 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
1902 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
1904 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
1908 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
1915 WaitcntBrackets &ScoreBrackets,
1919 if (OldWaitcntInstr)
1923 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
1927 ScoreBrackets.applyWaitcnt(
Wait);
1930 if (
Wait.ExpCnt != ~0u && It !=
Block.instr_end() &&
1933 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1941 <<
"Update Instr: " << *It);
1944 if (WCG->createNewWaitcnt(
Block, It,
Wait))
1953bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(
const MachineInstr &
MI)
const {
1961 if (
MI.memoperands_empty())
1970 unsigned AS = Memop->getAddrSpace();
1981bool SIInsertWaitcnts::mayAccessLDSThroughFlat(
const MachineInstr &
MI)
const {
1985 if (!
TII->usesLGKM_CNT(
MI))
1989 if (
ST->isTgSplitEnabled())
1994 if (
MI.memoperands_empty())
1999 unsigned AS = Memop->getAddrSpace();
2009bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
2014 if (
TII->isFLATScratch(
MI))
2018 if (
TII->isFLATGlobal(
MI))
2023 if (
MI.memoperands_empty())
2028 unsigned AS = Memop->getAddrSpace();
2029 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
2035 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
2036 Opc == AMDGPU::GLOBAL_WBINV;
2039void SIInsertWaitcnts::updateEventWaitcntAfter(
MachineInstr &Inst,
2040 WaitcntBrackets *ScoreBrackets) {
2046 if (
TII->isDS(Inst) &&
TII->usesLGKM_CNT(Inst)) {
2048 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
2049 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_ACCESS, Inst);
2050 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_GPR_LOCK, Inst);
2052 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2054 }
else if (
TII->isFLAT(Inst)) {
2061 int FlatASCount = 0;
2063 if (mayAccessVMEMThroughFlat(Inst)) {
2065 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2069 if (mayAccessLDSThroughFlat(Inst)) {
2071 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2080 if (FlatASCount > 1)
2081 ScoreBrackets->setPendingFlat();
2084 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2087 if (
ST->vmemWriteNeedsExpWaitcnt() &&
2089 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, VMW_GPR_LOCK, Inst);
2091 }
else if (
TII->isSMRD(Inst)) {
2092 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2093 }
else if (Inst.
isCall()) {
2096 ScoreBrackets->applyWaitcnt(
2097 WCG->getAllZeroWaitcnt(
false));
2098 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2104 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_LDS_ACCESS, Inst);
2105 }
else if (
TII->isVINTERP(Inst)) {
2106 int64_t
Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2107 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
2109 unsigned Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
2111 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_PARAM_ACCESS, Inst);
2113 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_POS_ACCESS, Inst);
2115 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_GPR_LOCK, Inst);
2118 case AMDGPU::S_SENDMSG:
2119 case AMDGPU::S_SENDMSG_RTN_B32:
2120 case AMDGPU::S_SENDMSG_RTN_B64:
2121 case AMDGPU::S_SENDMSGHALT:
2122 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SQ_MESSAGE, Inst);
2124 case AMDGPU::S_MEMTIME:
2125 case AMDGPU::S_MEMREALTIME:
2126 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
2127 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
2128 case AMDGPU::S_BARRIER_LEAVE:
2129 case AMDGPU::S_GET_BARRIER_STATE_M0:
2130 case AMDGPU::S_GET_BARRIER_STATE_IMM:
2131 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2137bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
2138 unsigned OtherScore) {
2139 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
2140 unsigned OtherShifted =
2141 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
2142 Score = std::max(MyShifted, OtherShifted);
2143 return OtherShifted > MyShifted;
2151bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
2152 bool StrictDom =
false;
2154 VgprUB = std::max(VgprUB,
Other.VgprUB);
2155 SgprUB = std::max(SgprUB,
Other.SgprUB);
2157 for (
auto T : inst_counter_types(MaxCounter)) {
2159 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[
T];
2160 const unsigned OtherEvents =
Other.PendingEvents & WaitEventMaskForInst[
T];
2161 if (OtherEvents & ~OldEvents)
2163 PendingEvents |= OtherEvents;
2166 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
2167 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
2168 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
2169 if (NewUB < ScoreLBs[
T])
2173 M.OldLB = ScoreLBs[
T];
2174 M.OtherLB =
Other.ScoreLBs[
T];
2175 M.MyShift = NewUB - ScoreUBs[
T];
2176 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
2178 ScoreUBs[
T] = NewUB;
2180 StrictDom |= mergeScore(M, LastFlat[
T],
Other.LastFlat[
T]);
2182 for (
int J = 0; J <= VgprUB; J++)
2183 StrictDom |= mergeScore(M, VgprScores[
T][J],
Other.VgprScores[
T][J]);
2185 if (
T == SmemAccessCounter) {
2186 for (
int J = 0; J <= SgprUB; J++)
2187 StrictDom |= mergeScore(M, SgprScores[J],
Other.SgprScores[J]);
2191 for (
int J = 0; J <= VgprUB; J++) {
2192 unsigned char NewVmemTypes = VgprVmemTypes[J] |
Other.VgprVmemTypes[J];
2193 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
2194 VgprVmemTypes[J] = NewVmemTypes;
2202 return Opcode == AMDGPU::S_WAITCNT ||
2205 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2206 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2213 WaitcntBrackets &ScoreBrackets) {
2217 dbgs() <<
"*** Block" <<
Block.getNumber() <<
" ***";
2218 ScoreBrackets.dump();
2224 bool VCCZCorrect =
true;
2225 if (
ST->hasReadVCCZBug()) {
2228 VCCZCorrect =
false;
2229 }
else if (!
ST->partialVCCWritesUpdateVCCZ()) {
2232 VCCZCorrect =
false;
2239 E =
Block.instr_end();
2246 if (!OldWaitcntInstr)
2247 OldWaitcntInstr = &Inst;
2252 bool FlushVmCnt =
Block.getFirstTerminator() == Inst &&
2253 isPreheaderToFlush(
Block, ScoreBrackets);
2256 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
2258 OldWaitcntInstr =
nullptr;
2261 bool RestoreVCCZ = !VCCZCorrect &&
readsVCCZ(Inst);
2264 if (
ST->hasReadVCCZBug() || !
ST->partialVCCWritesUpdateVCCZ()) {
2268 if (!
ST->partialVCCWritesUpdateVCCZ())
2269 VCCZCorrect =
false;
2278 if (
ST->hasReadVCCZBug() &&
2279 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
2282 VCCZCorrect =
false;
2290 if (
TII->isSMRD(Inst)) {
2294 if (!Memop->isInvariant()) {
2295 const Value *
Ptr = Memop->getValue();
2299 if (
ST->hasReadVCCZBug()) {
2301 VCCZCorrect =
false;
2305 updateEventWaitcntAfter(Inst, &ScoreBrackets);
2311 if (RequireCheckResourceType(Inst, context)) {
2313 ScoreBrackets->setScoreLB(LOAD_CNT,
2314 ScoreBrackets->getScoreUB(LOAD_CNT));
2321 ScoreBrackets.simplifyWaitcnt(
Wait);
2323 ScoreBrackets,
nullptr);
2328 ScoreBrackets.dump();
2338 TII->get(
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
2351 if (
Block.getFirstTerminator() ==
Block.end() &&
2352 isPreheaderToFlush(
Block, ScoreBrackets)) {
2353 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
2355 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
2357 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
2371 WaitcntBrackets &ScoreBrackets) {
2372 auto [Iterator, IsInserted] = PreheadersToFlush.
try_emplace(&
MBB,
false);
2374 return Iterator->second;
2385 shouldFlushVmCnt(
Loop, ScoreBrackets)) {
2386 Iterator->second =
true;
2393bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
2407 WaitcntBrackets &Brackets) {
2408 bool HasVMemLoad =
false;
2409 bool HasVMemStore =
false;
2410 bool UsesVgprLoadedOutside =
false;
2416 if (isVMEMOrFlatVMEM(
MI)) {
2420 HasVMemStore =
true;
2422 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++) {
2424 if (!
Op.isReg() || !
TRI->isVectorRegister(*
MRI,
Op.getReg()))
2437 if (Brackets.getRegScore(RegNo, LOAD_CNT) >
2438 Brackets.getScoreLB(LOAD_CNT) ||
2439 Brackets.getRegScore(RegNo, SAMPLE_CNT) >
2440 Brackets.getScoreLB(SAMPLE_CNT) ||
2441 Brackets.getRegScore(RegNo, BVH_CNT) >
2442 Brackets.getScoreLB(BVH_CNT)) {
2443 UsesVgprLoadedOutside =
true;
2449 else if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad() &&
Op.isDef())
2460 if (!
ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
2462 return HasVMemLoad && UsesVgprLoadedOutside;
2467 TII =
ST->getInstrInfo();
2468 TRI = &
TII->getRegisterInfo();
2471 MLI = &getAnalysis<MachineLoopInfo>();
2472 PDT = &getAnalysis<MachinePostDominatorTree>();
2473 if (
auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
2474 AA = &AAR->getAAResults();
2478 if (
ST->hasExtendedWaitCounts()) {
2479 MaxCounter = NUM_EXTENDED_INST_CNTS;
2480 WCGGFX12Plus = WaitcntGeneratorGFX12Plus(MF, MaxCounter);
2481 WCG = &WCGGFX12Plus;
2483 MaxCounter = NUM_NORMAL_INST_CNTS;
2484 WCGPreGFX12 = WaitcntGeneratorPreGFX12(MF);
2489 for (
auto T : inst_counter_types())
2490 ForceEmitWaitcnt[
T] =
false;
2492 const unsigned *WaitEventMaskForInst = WCG->getWaitEventMask();
2494 SmemAccessCounter = eventCounter(WaitEventMaskForInst, SMEM_ACCESS);
2496 HardwareLimits Limits = {};
2497 if (
ST->hasExtendedWaitCounts()) {
2510 unsigned NumVGPRsMax =
ST->getAddressableNumVGPRs();
2511 unsigned NumSGPRsMax =
ST->getAddressableNumSGPRs();
2512 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
2513 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
2515 RegisterEncoding Encoding = {};
2518 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
2521 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
2537 I != E && (
I->isPHI() ||
I->isMetaInstruction()); ++
I)
2540 if (
ST->hasExtendedWaitCounts()) {
2543 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
2544 if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
2548 TII->get(instrsForExtendedCounterTypes[CT]))
2555 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
2556 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2558 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
2559 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
2569 std::unique_ptr<WaitcntBrackets> Brackets;
2574 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
2577 BlockInfo &BI = BII->second;
2583 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
2585 *Brackets = *BI.Incoming;
2588 Brackets = std::make_unique<WaitcntBrackets>(
2589 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2592 *Brackets = WaitcntBrackets(ST, MaxCounter, Limits, Encoding,
2593 WaitEventMaskForInst, SmemAccessCounter);
2596 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
2599 if (Brackets->hasPendingEvent()) {
2600 BlockInfo *MoveBracketsToSucc =
nullptr;
2602 auto SuccBII = BlockInfos.
find(Succ);
2603 BlockInfo &SuccBI = SuccBII->second;
2604 if (!SuccBI.Incoming) {
2605 SuccBI.Dirty =
true;
2608 if (!MoveBracketsToSucc) {
2609 MoveBracketsToSucc = &SuccBI;
2611 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
2613 }
else if (SuccBI.Incoming->merge(*Brackets)) {
2614 SuccBI.Dirty =
true;
2619 if (MoveBracketsToSucc)
2620 MoveBracketsToSucc->Incoming = std::move(Brackets);
2625 if (
ST->hasScalarStores()) {
2627 bool HaveScalarStores =
false;
2631 if (!HaveScalarStores &&
TII->isScalarStore(
MI))
2632 HaveScalarStores =
true;
2634 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
2635 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
2640 if (HaveScalarStores) {
2650 bool SeenDCacheWB =
false;
2654 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
2655 SeenDCacheWB =
true;
2656 else if (
TII->isScalarStore(*
I))
2657 SeenDCacheWB =
false;
2660 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
2661 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
2674 if (
ST->requiresNopBeforeDeallocVGPRs()) {
2679 TII->get(AMDGPU::S_SENDMSG))
2683 ReleaseVGPRInsts.clear();
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Analysis containing CSE Info
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
static Function * getFunction(Constant *C)
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool isCacheInvOrWBInst(MachineInstr &Inst)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static std::optional< InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
static bool readsVCCZ(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isEntryFunction() const
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Represents a single loop in the control flow graph.
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Instructions::iterator instr_iterator
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
unsigned getNumOperands() const
Retuns the total number of operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isEXP(const MachineInstr &MI)
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
static bool isVIMAGE(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isVSAMPLE(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned getStorecntBitMask(const IsaVersion &Version)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getKmcntBitMask(const IsaVersion &Version)
unsigned getVmcntBitMask(const IsaVersion &Version)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getBvhcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
static unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt, unsigned Dscnt)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getLoadcntBitMask(const IsaVersion &Version)
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt, unsigned Dscnt)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Represents the counter values to wait for in an s_waitcnt instruction.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static constexpr bool is_iterable