42#define DEBUG_TYPE "si-insert-waitcnts"
45 "Force emit s_waitcnt expcnt(0) instrs");
47 "Force emit s_waitcnt lgkmcnt(0) instrs");
49 "Force emit s_waitcnt vmcnt(0) instrs");
52 "amdgpu-waitcnt-forcezero",
53 cl::desc(
"Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
67 SAMPLE_CNT = NUM_NORMAL_INST_CNTS,
70 NUM_EXTENDED_INST_CNTS,
71 NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
85auto inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS) {
86 return enum_seq(LOAD_CNT, MaxCounter);
89using RegInterval = std::pair<int, int>;
91struct HardwareLimits {
96 unsigned SamplecntMax;
101struct RegisterEncoding {
111 VMEM_SAMPLER_READ_ACCESS,
112 VMEM_BVH_READ_ACCESS,
114 SCRATCH_WRITE_ACCESS,
134enum RegisterMapping {
135 SQ_MAX_PGM_VGPRS = 512,
137 SQ_MAX_PGM_SGPRS = 256,
145 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS,
166static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
177static bool isNormalMode(InstCounterType MaxCounter) {
178 return MaxCounter == NUM_NORMAL_INST_CNTS;
183 assert(updateVMCntOnly(Inst));
186 return VMEM_NOSAMPLER;
190 return BaseInfo->
BVH ? VMEM_BVH
191 : BaseInfo->
Sampler ? VMEM_SAMPLER : VMEM_NOSAMPLER;
203 return Wait.StoreCnt;
205 return Wait.SampleCnt;
216 unsigned &WC = getCounterRef(
Wait,
T);
217 WC = std::min(WC, Count);
221 getCounterRef(
Wait,
T) = ~0
u;
225 return getCounterRef(
Wait,
T);
229InstCounterType eventCounter(
const unsigned *masks, WaitEventType E) {
230 for (
auto T : inst_counter_types()) {
231 if (masks[
T] & (1 << E))
245class WaitcntBrackets {
247 WaitcntBrackets(
const GCNSubtarget *SubTarget, InstCounterType MaxCounter,
248 HardwareLimits Limits, RegisterEncoding Encoding,
249 const unsigned *WaitEventMaskForInst,
250 InstCounterType SmemAccessCounter)
251 :
ST(SubTarget), MaxCounter(MaxCounter), Limits(Limits),
252 Encoding(Encoding), WaitEventMaskForInst(WaitEventMaskForInst),
253 SmemAccessCounter(SmemAccessCounter) {}
255 unsigned getWaitCountMax(InstCounterType
T)
const {
258 return Limits.LoadcntMax;
260 return Limits.DscntMax;
262 return Limits.ExpcntMax;
264 return Limits.StorecntMax;
266 return Limits.SamplecntMax;
268 return Limits.BvhcntMax;
270 return Limits.KmcntMax;
277 unsigned getScoreLB(InstCounterType
T)
const {
282 unsigned getScoreUB(InstCounterType
T)
const {
287 unsigned getScoreRange(InstCounterType
T)
const {
288 return getScoreUB(
T) - getScoreLB(
T);
291 unsigned getRegScore(
int GprNo, InstCounterType
T)
const {
292 if (GprNo < NUM_ALL_VGPRS) {
293 return VgprScores[
T][GprNo];
295 assert(
T == SmemAccessCounter);
296 return SgprScores[GprNo - NUM_ALL_VGPRS];
305 bool counterOutOfOrder(InstCounterType
T)
const;
307 void simplifyWaitcnt(InstCounterType
T,
unsigned &Count)
const;
310 void applyWaitcnt(InstCounterType
T,
unsigned Count);
315 unsigned hasPendingEvent()
const {
return PendingEvents; }
316 unsigned hasPendingEvent(WaitEventType E)
const {
317 return PendingEvents & (1 << E);
319 unsigned hasPendingEvent(InstCounterType
T)
const {
320 unsigned HasPending = PendingEvents & WaitEventMaskForInst[
T];
321 assert((HasPending != 0) == (getScoreRange(
T) != 0));
325 bool hasMixedPendingEvents(InstCounterType
T)
const {
326 unsigned Events = hasPendingEvent(
T);
328 return Events & (Events - 1);
331 bool hasPendingFlat()
const {
332 return ((LastFlat[DS_CNT] > ScoreLBs[DS_CNT] &&
333 LastFlat[DS_CNT] <= ScoreUBs[DS_CNT]) ||
334 (LastFlat[LOAD_CNT] > ScoreLBs[LOAD_CNT] &&
335 LastFlat[LOAD_CNT] <= ScoreUBs[LOAD_CNT]));
338 void setPendingFlat() {
339 LastFlat[LOAD_CNT] = ScoreUBs[LOAD_CNT];
340 LastFlat[DS_CNT] = ScoreUBs[DS_CNT];
345 bool hasOtherPendingVmemTypes(
int GprNo, VmemType V)
const {
346 assert(GprNo < NUM_ALL_VGPRS);
347 return VgprVmemTypes[GprNo] & ~(1 <<
V);
350 void clearVgprVmemTypes(
int GprNo) {
351 assert(GprNo < NUM_ALL_VGPRS);
352 VgprVmemTypes[GprNo] = 0;
355 void setStateOnFunctionEntryOrReturn() {
356 setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getWaitCountMax(STORE_CNT));
357 PendingEvents |= WaitEventMaskForInst[STORE_CNT];
374 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
375 unsigned OtherScore);
377 void setScoreLB(InstCounterType
T,
unsigned Val) {
382 void setScoreUB(InstCounterType
T,
unsigned Val) {
389 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
393 void setRegScore(
int GprNo, InstCounterType
T,
unsigned Val) {
394 if (GprNo < NUM_ALL_VGPRS) {
395 VgprUB = std::max(VgprUB, GprNo);
396 VgprScores[
T][GprNo] = Val;
398 assert(
T == SmemAccessCounter);
399 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
400 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
406 unsigned OpNo,
unsigned Val);
409 InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
410 HardwareLimits Limits = {};
411 RegisterEncoding Encoding = {};
412 const unsigned *WaitEventMaskForInst;
413 InstCounterType SmemAccessCounter;
414 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
415 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
416 unsigned PendingEvents = 0;
418 unsigned LastFlat[NUM_INST_CNTS] = {0};
423 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
426 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
429 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
441class WaitcntGenerator {
446 InstCounterType MaxCounter;
449 WaitcntGenerator() {}
450 WaitcntGenerator(
const GCNSubtarget *ST, InstCounterType MaxCounter)
466 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
481 virtual const unsigned *getWaitEventMask()
const = 0;
485 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const = 0;
487 virtual ~WaitcntGenerator() =
default;
490 static constexpr unsigned
491 eventMask(std::initializer_list<WaitEventType> Events) {
493 for (
auto &E : Events)
500class WaitcntGeneratorPreGFX12 :
public WaitcntGenerator {
502 WaitcntGeneratorPreGFX12() {}
504 : WaitcntGenerator(
ST, NUM_NORMAL_INST_CNTS) {}
507 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
515 const unsigned *getWaitEventMask()
const override {
518 static const unsigned WaitEventMaskForInstPreGFX12[NUM_INST_CNTS] = {
519 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS,
520 VMEM_BVH_READ_ACCESS}),
521 eventMask({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
522 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
523 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
524 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
529 return WaitEventMaskForInstPreGFX12;
532 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
535class WaitcntGeneratorGFX12Plus :
public WaitcntGenerator {
537 WaitcntGeneratorGFX12Plus() {}
538 WaitcntGeneratorGFX12Plus(
const GCNSubtarget *ST, InstCounterType MaxCounter)
539 : WaitcntGenerator(
ST, MaxCounter) {}
542 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
550 const unsigned *getWaitEventMask()
const override {
553 static const unsigned WaitEventMaskForInstGFX12Plus[NUM_INST_CNTS] = {
554 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS}),
555 eventMask({LDS_ACCESS, GDS_ACCESS}),
556 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
557 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
558 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
559 eventMask({VMEM_SAMPLER_READ_ACCESS}),
560 eventMask({VMEM_BVH_READ_ACCESS}),
561 eventMask({SMEM_ACCESS, SQ_MESSAGE})};
563 return WaitEventMaskForInstGFX12Plus;
566 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
583 std::unique_ptr<WaitcntBrackets>
Incoming;
587 InstCounterType SmemAccessCounter;
593 bool ForceEmitZeroWaitcnts;
594 bool ForceEmitWaitcnt[NUM_INST_CNTS];
601 WaitcntGeneratorPreGFX12 WCGPreGFX12;
602 WaitcntGeneratorGFX12Plus WCGGFX12Plus;
604 WaitcntGenerator *WCG =
nullptr;
610 InstCounterType MaxCounter = NUM_NORMAL_INST_CNTS;
616 (void)ForceExpCounter;
617 (void)ForceLgkmCounter;
618 (void)ForceVMCounter;
621 bool shouldFlushVmCnt(
MachineLoop *
ML, WaitcntBrackets &Brackets);
623 WaitcntBrackets &ScoreBrackets);
628 return "SI insert wait instructions";
640 bool isForceEmitWaitcnt()
const {
641 for (
auto T : inst_counter_types())
642 if (ForceEmitWaitcnt[
T])
647 void setForceEmitWaitcnt() {
653 ForceEmitWaitcnt[
EXP_CNT] =
true;
655 ForceEmitWaitcnt[
EXP_CNT] =
false;
660 ForceEmitWaitcnt[DS_CNT] =
true;
661 ForceEmitWaitcnt[KM_CNT] =
true;
663 ForceEmitWaitcnt[DS_CNT] =
false;
664 ForceEmitWaitcnt[KM_CNT] =
false;
669 ForceEmitWaitcnt[LOAD_CNT] =
true;
670 ForceEmitWaitcnt[SAMPLE_CNT] =
true;
671 ForceEmitWaitcnt[BVH_CNT] =
true;
673 ForceEmitWaitcnt[LOAD_CNT] =
false;
674 ForceEmitWaitcnt[SAMPLE_CNT] =
false;
675 ForceEmitWaitcnt[BVH_CNT] =
false;
682 WaitEventType getVmemWaitEventType(
const MachineInstr &Inst)
const {
684 static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
685 VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
696 return SCRATCH_WRITE_ACCESS;
697 return VMEM_WRITE_ACCESS;
700 return VMEM_READ_ACCESS;
701 return VmemReadMapping[getVmemType(Inst)];
708 WaitcntBrackets &ScoreBrackets,
716 WaitcntBrackets *ScoreBrackets);
718 WaitcntBrackets &ScoreBrackets);
723RegInterval WaitcntBrackets::getRegInterval(
const MachineInstr *
MI,
726 unsigned OpNo)
const {
728 if (!
TRI->isInAllocatableClass(
Op.getReg()))
740 if (
TRI->isVectorRegister(*
MRI,
Op.getReg())) {
741 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
744 Result.first += AGPR_OFFSET;
746 }
else if (
TRI->isSGPRReg(*
MRI,
Op.getReg())) {
747 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
748 Result.first =
Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
750 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
758 unsigned Size =
TRI->getRegSizeInBits(*RC);
770 assert(
TRI->isVectorRegister(*
MRI,
MI->getOperand(OpNo).getReg()));
772 setRegScore(RegNo, EXP_CNT, Val);
780 InstCounterType
T = eventCounter(WaitEventMaskForInst, E);
782 unsigned UB = getScoreUB(
T);
783 unsigned CurrScore = UB + 1;
789 PendingEvents |= 1 << E;
790 setScoreUB(
T, CurrScore);
800 if (AddrOpIdx != -1) {
801 setExpScore(&Inst,
TII,
TRI,
MRI, AddrOpIdx, CurrScore);
814 AMDGPU::OpName::data1),
819 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
820 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
823 if (
Op.isReg() && !
Op.isDef() &&
824 TRI->isVectorRegister(*
MRI,
Op.getReg())) {
825 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
829 }
else if (
TII->isFLAT(Inst)) {
841 }
else if (
TII->isMIMG(Inst)) {
843 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
850 }
else if (
TII->isMTBUF(Inst)) {
852 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
854 }
else if (
TII->isMUBUF(Inst)) {
856 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
863 }
else if (
TII->isLDSDIR(Inst)) {
870 if (
TII->isEXP(Inst)) {
889 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
894 }
else if (Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORD ||
895 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 ||
896 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) {
902 setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore);
909 if (!
Op.isReg() || !
Op.isDef())
912 if (
T == LOAD_CNT ||
T == SAMPLE_CNT ||
T == BVH_CNT) {
913 if (
Interval.first >= NUM_ALL_VGPRS)
915 if (updateVMCntOnly(Inst)) {
920 VmemType
V = getVmemType(Inst);
922 VgprVmemTypes[RegNo] |= 1 <<
V;
926 setRegScore(RegNo,
T, CurrScore);
930 (
TII->isDS(Inst) ||
TII->mayWriteLDSThroughDMA(Inst))) {
935 if (!
MemOp->isStore() ||
940 auto AAI =
MemOp->getAAInfo();
948 if (!AAI || !AAI.Scope)
950 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E && !Slot; ++
I) {
951 for (
const auto *
MemOp : LDSDMAStores[
I]->memoperands()) {
952 if (
MemOp->isStore() && AAI ==
MemOp->getAAInfo()) {
958 if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
960 LDSDMAStores.push_back(&Inst);
961 Slot = LDSDMAStores.size();
964 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot,
T, CurrScore);
966 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS,
T, CurrScore);
973 for (
auto T : inst_counter_types(MaxCounter)) {
974 unsigned SR = getScoreRange(
T);
978 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"LOAD" :
"VM") <<
"_CNT("
982 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"DS" :
"LGKM") <<
"_CNT("
986 OS <<
" EXP_CNT(" << SR <<
"): ";
989 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"STORE" :
"VS") <<
"_CNT("
993 OS <<
" SAMPLE_CNT(" << SR <<
"): ";
996 OS <<
" BVH_CNT(" << SR <<
"): ";
999 OS <<
" KM_CNT(" << SR <<
"): ";
1002 OS <<
" UNKNOWN(" << SR <<
"): ";
1008 unsigned LB = getScoreLB(
T);
1010 for (
int J = 0; J <= VgprUB; J++) {
1011 unsigned RegScore = getRegScore(J,
T);
1014 unsigned RelScore = RegScore - LB - 1;
1015 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
1016 OS << RelScore <<
":v" << J <<
" ";
1018 OS << RelScore <<
":ds ";
1022 if (
T == SmemAccessCounter) {
1023 for (
int J = 0; J <= SgprUB; J++) {
1024 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS,
T);
1027 unsigned RelScore = RegScore - LB - 1;
1028 OS << RelScore <<
":s" << J <<
" ";
1040 simplifyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1041 simplifyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1042 simplifyWaitcnt(DS_CNT,
Wait.DsCnt);
1043 simplifyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1044 simplifyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1045 simplifyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1046 simplifyWaitcnt(KM_CNT,
Wait.KmCnt);
1049void WaitcntBrackets::simplifyWaitcnt(InstCounterType
T,
1050 unsigned &Count)
const {
1054 if (Count >= getScoreRange(
T))
1058void WaitcntBrackets::determineWait(InstCounterType
T,
int RegNo,
1060 unsigned ScoreToWait = getRegScore(RegNo,
T);
1064 const unsigned LB = getScoreLB(
T);
1065 const unsigned UB = getScoreUB(
T);
1066 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1067 if ((
T == LOAD_CNT ||
T == DS_CNT) && hasPendingFlat() &&
1068 !
ST->hasFlatLgkmVMemCountInOrder()) {
1072 addWait(
Wait,
T, 0);
1073 }
else if (counterOutOfOrder(
T)) {
1077 addWait(
Wait,
T, 0);
1081 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(
T) - 1);
1082 addWait(
Wait,
T, NeededWait);
1088 applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1089 applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1090 applyWaitcnt(DS_CNT,
Wait.DsCnt);
1091 applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1092 applyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1093 applyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1094 applyWaitcnt(KM_CNT,
Wait.KmCnt);
1097void WaitcntBrackets::applyWaitcnt(InstCounterType
T,
unsigned Count) {
1098 const unsigned UB = getScoreUB(
T);
1102 if (counterOutOfOrder(
T))
1104 setScoreLB(
T, std::max(getScoreLB(
T), UB - Count));
1107 PendingEvents &= ~WaitEventMaskForInst[
T];
1113bool WaitcntBrackets::counterOutOfOrder(InstCounterType
T)
const {
1115 if (
T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
1117 return hasMixedPendingEvents(
T);
1127char SIInsertWaitcnts::
ID = 0;
1132 return new SIInsertWaitcnts();
1142 if (NewEnc == MO.
getImm())
1153 case AMDGPU::S_WAIT_LOADCNT:
1155 case AMDGPU::S_WAIT_EXPCNT:
1157 case AMDGPU::S_WAIT_STORECNT:
1159 case AMDGPU::S_WAIT_SAMPLECNT:
1161 case AMDGPU::S_WAIT_BVHCNT:
1163 case AMDGPU::S_WAIT_DSCNT:
1165 case AMDGPU::S_WAIT_KMCNT:
1172bool WaitcntGenerator::promoteSoftWaitCnt(
MachineInstr *Waitcnt)
const {
1186bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1187 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1190 assert(isNormalMode(MaxCounter));
1198 if (II.isMetaInstruction())
1202 bool IsSoft = Opcode != II.getOpcode();
1206 if (Opcode == AMDGPU::S_WAITCNT) {
1207 unsigned IEnc = II.getOperand(0).getImm();
1210 ScoreBrackets.simplifyWaitcnt(OldWait);
1214 if (WaitcntInstr || (!
Wait.hasWaitExceptStoreCnt() && IsSoft)) {
1215 II.eraseFromParent();
1220 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1221 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1224 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1226 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt);
1227 Wait.StoreCnt = std::min(
Wait.StoreCnt, OldVSCnt);
1229 if (WaitcntVsCntInstr || (!
Wait.hasWaitStoreCnt() && IsSoft)) {
1230 II.eraseFromParent();
1233 WaitcntVsCntInstr = &II;
1240 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1242 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1243 ScoreBrackets.applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1244 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1251 <<
"applyPreexistingWaitcnt\n"
1252 <<
"New Instr at block end: " << *WaitcntInstr <<
'\n'
1253 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1254 <<
"Old Instr: " << *It
1255 <<
"New Instr: " << *WaitcntInstr <<
'\n');
1258 if (WaitcntVsCntInstr) {
1260 AMDGPU::OpName::simm16,
Wait.StoreCnt);
1261 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1263 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1264 Wait.StoreCnt = ~0
u;
1267 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1268 <<
"New Instr at block end: " << *WaitcntVsCntInstr
1270 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1271 <<
"Old Instr: " << *It
1272 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
1280bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1284 assert(isNormalMode(MaxCounter));
1291 if (
Wait.hasWaitExceptStoreCnt()) {
1293 [[maybe_unused]]
auto SWaitInst =
1298 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1299 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1302 if (
Wait.hasWaitStoreCnt()) {
1305 [[maybe_unused]]
auto SWaitInst =
1312 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1313 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1320WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1325WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1333bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1334 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1337 assert(!isNormalMode(MaxCounter));
1346 if (II.isMetaInstruction())
1355 bool IsSoft = Opcode != II.getOpcode();
1357 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1359 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1362 ScoreBrackets.simplifyWaitcnt(OldWait);
1364 UpdatableInstr = &CombinedLoadDsCntInstr;
1365 }
else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1367 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1370 ScoreBrackets.simplifyWaitcnt(OldWait);
1372 UpdatableInstr = &CombinedStoreDsCntInstr;
1377 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1379 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt);
1380 addWait(
Wait, CT.value(), OldCnt);
1381 UpdatableInstr = &WaitInstrs[CT.value()];
1385 if (!*UpdatableInstr) {
1386 *UpdatableInstr = &II;
1393 if (CombinedLoadDsCntInstr) {
1401 if (
Wait.LoadCnt != ~0u &&
Wait.DsCnt != ~0u) {
1404 AMDGPU::OpName::simm16, NewEnc);
1405 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
1406 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1407 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1412 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1413 <<
"New Instr at block end: "
1414 << *CombinedLoadDsCntInstr <<
'\n'
1415 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1416 <<
"Old Instr: " << *It <<
"New Instr: "
1417 << *CombinedLoadDsCntInstr <<
'\n');
1424 if (CombinedStoreDsCntInstr) {
1426 if (
Wait.StoreCnt != ~0u &&
Wait.DsCnt != ~0u) {
1429 AMDGPU::OpName::simm16, NewEnc);
1430 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
1431 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1432 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1433 Wait.StoreCnt = ~0
u;
1437 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1438 <<
"New Instr at block end: "
1439 << *CombinedStoreDsCntInstr <<
'\n'
1440 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1441 <<
"Old Instr: " << *It <<
"New Instr: "
1442 << *CombinedStoreDsCntInstr <<
'\n');
1455 if (
Wait.DsCnt != ~0u) {
1464 if (
Wait.LoadCnt != ~0u) {
1465 WaitsToErase.
push_back(&WaitInstrs[LOAD_CNT]);
1466 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1467 }
else if (
Wait.StoreCnt != ~0u) {
1468 WaitsToErase.
push_back(&WaitInstrs[STORE_CNT]);
1469 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1476 (*WI)->eraseFromParent();
1482 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1483 if (!WaitInstrs[CT])
1486 unsigned NewCnt = getWait(
Wait, CT);
1487 if (NewCnt != ~0u) {
1489 AMDGPU::OpName::simm16, NewCnt);
1490 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
1492 ScoreBrackets.applyWaitcnt(CT, NewCnt);
1493 setNoWait(
Wait, CT);
1496 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1497 <<
"New Instr at block end: " << *WaitInstrs[CT]
1499 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1500 <<
"Old Instr: " << *It
1501 <<
"New Instr: " << *WaitInstrs[CT] <<
'\n');
1512bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
1516 assert(!isNormalMode(MaxCounter));
1522 if (
Wait.DsCnt != ~0u) {
1525 if (
Wait.LoadCnt != ~0u) {
1533 }
else if (
Wait.StoreCnt != ~0u) {
1540 Wait.StoreCnt = ~0
u;
1548 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1549 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1556 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1557 unsigned Count = getWait(
Wait, CT);
1561 [[maybe_unused]]
auto SWaitInst =
1568 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1569 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1576 unsigned Opc =
MI.getOpcode();
1577 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
1578 !
MI.getOperand(1).isUndef();
1608bool SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &
MI,
1609 WaitcntBrackets &ScoreBrackets,
1612 setForceEmitWaitcnt();
1614 if (
MI.isMetaInstruction())
1623 if (
MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1624 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1625 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1626 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1627 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1634 if (
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1635 MI.getOpcode() == AMDGPU::SI_RETURN ||
1636 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1638 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
false));
1646 else if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1647 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1649 ScoreBrackets.getScoreRange(STORE_CNT) != 0 &&
1650 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1654 else if ((
MI.getOpcode() == AMDGPU::S_SENDMSG ||
1655 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1656 ST->hasLegacyGeometry() &&
1662 else if (
MI.getOpcode() == SC_FENCE) {
1663 const unsigned int group_size =
1664 context->shader_info->GetMaxThreadGroupSize();
1666 const bool group_is_multi_wave =
1667 (group_size == 0 || group_size > target_info->GetWaveFrontSize());
1668 const bool fence_is_global = !((SCInstInternalMisc*)Inst)->IsGroupFence();
1670 for (
unsigned int i = 0; i < Inst->NumSrcOperands(); i++) {
1671 SCRegType src_type = Inst->GetSrcType(i);
1674 if (group_is_multi_wave ||
1675 context->OptFlagIsOn(OPT_R1100_LDSMEM_FENCE_CHICKEN_BIT)) {
1676 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1677 ScoreBrackets->getScoreUB(DS_CNT));
1679 if (target_info->HasBufferLoadToLDS()) {
1680 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1681 ScoreBrackets->getScoreUB(LOAD_CNT));
1687 if (group_is_multi_wave || fence_is_global) {
1688 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1689 ScoreBrackets->getScoreUB(EXP_CNT));
1690 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1691 ScoreBrackets->getScoreUB(DS_CNT));
1699 if (group_is_multi_wave || fence_is_global) {
1700 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1701 ScoreBrackets->getScoreUB(EXP_CNT));
1702 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1703 ScoreBrackets->getScoreUB(LOAD_CNT));
1720 if (
MI.modifiesRegister(AMDGPU::EXEC,
TRI)) {
1723 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1724 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1725 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1726 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1740 if (
MI.getOperand(CallAddrOpIdx).isReg()) {
1741 RegInterval CallAddrOpInterval =
1742 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, CallAddrOpIdx);
1744 for (
int RegNo = CallAddrOpInterval.first;
1745 RegNo < CallAddrOpInterval.second; ++RegNo)
1746 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1750 if (RtnAddrOpIdx != -1) {
1751 RegInterval RtnAddrOpInterval =
1752 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, RtnAddrOpIdx);
1754 for (
int RegNo = RtnAddrOpInterval.first;
1755 RegNo < RtnAddrOpInterval.second; ++RegNo)
1756 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1775 const Value *
Ptr = Memop->getValue();
1776 if (Memop->isStore() && SLoadAddresses.
count(
Ptr)) {
1777 addWait(
Wait, SmemAccessCounter, 0);
1781 unsigned AS = Memop->getAddrSpace();
1785 if (
TII->mayWriteLDSThroughDMA(
MI))
1789 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1790 bool FoundAliasingStore =
false;
1797 if (
Ptr && Memop->getAAInfo() && Memop->getAAInfo().Scope) {
1798 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
1799 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E; ++
I) {
1800 if (
MI.mayAlias(AA, *LDSDMAStores[
I],
true)) {
1801 FoundAliasingStore =
true;
1802 ScoreBrackets.determineWait(LOAD_CNT, RegNo +
I + 1,
Wait);
1806 if (!FoundAliasingStore)
1807 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1808 if (Memop->isStore()) {
1809 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1814 for (
unsigned I = 0, E =
MI.getNumOperands();
I != E; ++
I) {
1820 if (
Op.isTied() &&
Op.isUse() &&
TII->doesNotReadTiedSource(
MI))
1825 const bool IsVGPR =
TRI->isVectorRegister(*
MRI,
Op.getReg());
1832 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
1833 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1835 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1836 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo,
Wait);
1837 ScoreBrackets.determineWait(BVH_CNT, RegNo,
Wait);
1838 ScoreBrackets.clearVgprVmemTypes(RegNo);
1840 if (
Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1841 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1843 ScoreBrackets.determineWait(DS_CNT, RegNo,
Wait);
1845 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1856 if (
MI.getOpcode() == AMDGPU::S_BARRIER &&
1857 !
ST->hasAutoWaitcntBeforeBarrier() && !
ST->supportsBackOffBarrier()) {
1858 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
true));
1865 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1871 ScoreBrackets.simplifyWaitcnt(
Wait);
1873 if (ForceEmitZeroWaitcnts)
1874 Wait = WCG->getAllZeroWaitcnt(
false);
1876 if (ForceEmitWaitcnt[LOAD_CNT])
1878 if (ForceEmitWaitcnt[EXP_CNT])
1880 if (ForceEmitWaitcnt[DS_CNT])
1882 if (ForceEmitWaitcnt[SAMPLE_CNT])
1884 if (ForceEmitWaitcnt[BVH_CNT])
1886 if (ForceEmitWaitcnt[KM_CNT])
1890 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
1892 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
1894 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
1898 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
1905 WaitcntBrackets &ScoreBrackets,
1909 if (OldWaitcntInstr)
1913 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
1917 ScoreBrackets.applyWaitcnt(
Wait);
1920 if (
Wait.ExpCnt != ~0u && It !=
Block.instr_end() &&
1923 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1931 <<
"Update Instr: " << *It);
1934 if (WCG->createNewWaitcnt(
Block, It,
Wait))
1943bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(
const MachineInstr &
MI)
const {
1951 if (
MI.memoperands_empty())
1960 unsigned AS = Memop->getAddrSpace();
1971bool SIInsertWaitcnts::mayAccessLDSThroughFlat(
const MachineInstr &
MI)
const {
1975 if (!
TII->usesLGKM_CNT(
MI))
1979 if (
ST->isTgSplitEnabled())
1984 if (
MI.memoperands_empty())
1989 unsigned AS = Memop->getAddrSpace();
1999bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
2004 if (
TII->isFLATScratch(
MI))
2008 if (
TII->isFLATGlobal(
MI))
2013 if (
MI.memoperands_empty())
2018 unsigned AS = Memop->getAddrSpace();
2019 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
2025 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
2026 Opc == AMDGPU::GLOBAL_WBINV;
2029void SIInsertWaitcnts::updateEventWaitcntAfter(
MachineInstr &Inst,
2030 WaitcntBrackets *ScoreBrackets) {
2036 if (
TII->isDS(Inst) &&
TII->usesLGKM_CNT(Inst)) {
2038 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
2039 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_ACCESS, Inst);
2040 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_GPR_LOCK, Inst);
2042 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2044 }
else if (
TII->isFLAT(Inst)) {
2051 int FlatASCount = 0;
2053 if (mayAccessVMEMThroughFlat(Inst)) {
2055 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2059 if (mayAccessLDSThroughFlat(Inst)) {
2061 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2070 if (FlatASCount > 1)
2071 ScoreBrackets->setPendingFlat();
2074 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2077 if (
ST->vmemWriteNeedsExpWaitcnt() &&
2079 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, VMW_GPR_LOCK, Inst);
2081 }
else if (
TII->isSMRD(Inst)) {
2082 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2083 }
else if (Inst.
isCall()) {
2086 ScoreBrackets->applyWaitcnt(
2087 WCG->getAllZeroWaitcnt(
false));
2088 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2094 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_LDS_ACCESS, Inst);
2095 }
else if (
TII->isVINTERP(Inst)) {
2096 int64_t
Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2097 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
2099 unsigned Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
2101 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_PARAM_ACCESS, Inst);
2103 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_POS_ACCESS, Inst);
2105 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_GPR_LOCK, Inst);
2108 case AMDGPU::S_SENDMSG:
2109 case AMDGPU::S_SENDMSG_RTN_B32:
2110 case AMDGPU::S_SENDMSG_RTN_B64:
2111 case AMDGPU::S_SENDMSGHALT:
2112 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SQ_MESSAGE, Inst);
2114 case AMDGPU::S_MEMTIME:
2115 case AMDGPU::S_MEMREALTIME:
2116 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
2117 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
2118 case AMDGPU::S_BARRIER_LEAVE:
2119 case AMDGPU::S_GET_BARRIER_STATE_M0:
2120 case AMDGPU::S_GET_BARRIER_STATE_IMM:
2121 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2127bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
2128 unsigned OtherScore) {
2129 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
2130 unsigned OtherShifted =
2131 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
2132 Score = std::max(MyShifted, OtherShifted);
2133 return OtherShifted > MyShifted;
2141bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
2142 bool StrictDom =
false;
2144 VgprUB = std::max(VgprUB,
Other.VgprUB);
2145 SgprUB = std::max(SgprUB,
Other.SgprUB);
2147 for (
auto T : inst_counter_types(MaxCounter)) {
2149 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[
T];
2150 const unsigned OtherEvents =
Other.PendingEvents & WaitEventMaskForInst[
T];
2151 if (OtherEvents & ~OldEvents)
2153 PendingEvents |= OtherEvents;
2156 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
2157 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
2158 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
2159 if (NewUB < ScoreLBs[
T])
2163 M.OldLB = ScoreLBs[
T];
2164 M.OtherLB =
Other.ScoreLBs[
T];
2165 M.MyShift = NewUB - ScoreUBs[
T];
2166 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
2168 ScoreUBs[
T] = NewUB;
2170 StrictDom |= mergeScore(M, LastFlat[
T],
Other.LastFlat[
T]);
2172 for (
int J = 0; J <= VgprUB; J++)
2173 StrictDom |= mergeScore(M, VgprScores[
T][J],
Other.VgprScores[
T][J]);
2175 if (
T == SmemAccessCounter) {
2176 for (
int J = 0; J <= SgprUB; J++)
2177 StrictDom |= mergeScore(M, SgprScores[J],
Other.SgprScores[J]);
2181 for (
int J = 0; J <= VgprUB; J++) {
2182 unsigned char NewVmemTypes = VgprVmemTypes[J] |
Other.VgprVmemTypes[J];
2183 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
2184 VgprVmemTypes[J] = NewVmemTypes;
2192 return Opcode == AMDGPU::S_WAITCNT ||
2195 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2196 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2203 WaitcntBrackets &ScoreBrackets) {
2207 dbgs() <<
"*** Block" <<
Block.getNumber() <<
" ***";
2208 ScoreBrackets.dump();
2214 bool VCCZCorrect =
true;
2215 if (
ST->hasReadVCCZBug()) {
2218 VCCZCorrect =
false;
2219 }
else if (!
ST->partialVCCWritesUpdateVCCZ()) {
2222 VCCZCorrect =
false;
2229 E =
Block.instr_end();
2236 if (!OldWaitcntInstr)
2237 OldWaitcntInstr = &Inst;
2242 bool FlushVmCnt =
Block.getFirstTerminator() == Inst &&
2243 isPreheaderToFlush(
Block, ScoreBrackets);
2246 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
2248 OldWaitcntInstr =
nullptr;
2251 bool RestoreVCCZ = !VCCZCorrect &&
readsVCCZ(Inst);
2254 if (
ST->hasReadVCCZBug() || !
ST->partialVCCWritesUpdateVCCZ()) {
2258 if (!
ST->partialVCCWritesUpdateVCCZ())
2259 VCCZCorrect =
false;
2268 if (
ST->hasReadVCCZBug() &&
2269 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
2272 VCCZCorrect =
false;
2280 if (
TII->isSMRD(Inst)) {
2284 if (!Memop->isInvariant()) {
2285 const Value *
Ptr = Memop->getValue();
2289 if (
ST->hasReadVCCZBug()) {
2291 VCCZCorrect =
false;
2295 updateEventWaitcntAfter(Inst, &ScoreBrackets);
2301 if (RequireCheckResourceType(Inst, context)) {
2303 ScoreBrackets->setScoreLB(LOAD_CNT,
2304 ScoreBrackets->getScoreUB(LOAD_CNT));
2311 ScoreBrackets.simplifyWaitcnt(
Wait);
2313 ScoreBrackets,
nullptr);
2318 ScoreBrackets.dump();
2328 TII->get(
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
2341 if (
Block.getFirstTerminator() ==
Block.end() &&
2342 isPreheaderToFlush(
Block, ScoreBrackets)) {
2343 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
2345 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
2347 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
2361 WaitcntBrackets &ScoreBrackets) {
2362 auto [Iterator, IsInserted] = PreheadersToFlush.
try_emplace(&
MBB,
false);
2364 return Iterator->second;
2375 shouldFlushVmCnt(
Loop, ScoreBrackets)) {
2376 Iterator->second =
true;
2383bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
2397 WaitcntBrackets &Brackets) {
2398 bool HasVMemLoad =
false;
2399 bool HasVMemStore =
false;
2400 bool UsesVgprLoadedOutside =
false;
2406 if (isVMEMOrFlatVMEM(
MI)) {
2410 HasVMemStore =
true;
2412 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++) {
2414 if (!
Op.isReg() || !
TRI->isVectorRegister(*
MRI,
Op.getReg()))
2427 if (Brackets.getRegScore(RegNo, LOAD_CNT) >
2428 Brackets.getScoreLB(LOAD_CNT) ||
2429 Brackets.getRegScore(RegNo, SAMPLE_CNT) >
2430 Brackets.getScoreLB(SAMPLE_CNT) ||
2431 Brackets.getRegScore(RegNo, BVH_CNT) >
2432 Brackets.getScoreLB(BVH_CNT)) {
2433 UsesVgprLoadedOutside =
true;
2439 else if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad() &&
Op.isDef())
2450 if (!
ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
2452 return HasVMemLoad && UsesVgprLoadedOutside;
2457 TII =
ST->getInstrInfo();
2458 TRI = &
TII->getRegisterInfo();
2461 MLI = &getAnalysis<MachineLoopInfo>();
2462 PDT = &getAnalysis<MachinePostDominatorTree>();
2463 if (
auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
2464 AA = &AAR->getAAResults();
2468 if (
ST->hasExtendedWaitCounts()) {
2469 MaxCounter = NUM_EXTENDED_INST_CNTS;
2470 WCGGFX12Plus = WaitcntGeneratorGFX12Plus(ST, MaxCounter);
2471 WCG = &WCGGFX12Plus;
2473 MaxCounter = NUM_NORMAL_INST_CNTS;
2474 WCGPreGFX12 = WaitcntGeneratorPreGFX12(ST);
2479 for (
auto T : inst_counter_types())
2480 ForceEmitWaitcnt[
T] =
false;
2482 const unsigned *WaitEventMaskForInst = WCG->getWaitEventMask();
2484 SmemAccessCounter = eventCounter(WaitEventMaskForInst, SMEM_ACCESS);
2489 HardwareLimits Limits = {};
2490 if (
ST->hasExtendedWaitCounts()) {
2503 unsigned NumVGPRsMax =
ST->getAddressableNumVGPRs();
2504 unsigned NumSGPRsMax =
ST->getAddressableNumSGPRs();
2505 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
2506 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
2508 RegisterEncoding Encoding = {};
2511 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
2514 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
2530 I != E && (
I->isPHI() ||
I->isMetaInstruction()); ++
I)
2533 if (
ST->hasExtendedWaitCounts()) {
2536 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
2537 if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
2541 TII->get(instrsForExtendedCounterTypes[CT]))
2548 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
2549 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2551 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
2552 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
2562 std::unique_ptr<WaitcntBrackets> Brackets;
2567 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
2570 BlockInfo &BI = BII->second;
2576 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
2578 *Brackets = *BI.Incoming;
2581 Brackets = std::make_unique<WaitcntBrackets>(
2582 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2585 *Brackets = WaitcntBrackets(ST, MaxCounter, Limits, Encoding,
2586 WaitEventMaskForInst, SmemAccessCounter);
2589 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
2592 if (Brackets->hasPendingEvent()) {
2593 BlockInfo *MoveBracketsToSucc =
nullptr;
2595 auto SuccBII = BlockInfos.
find(Succ);
2596 BlockInfo &SuccBI = SuccBII->second;
2597 if (!SuccBI.Incoming) {
2598 SuccBI.Dirty =
true;
2601 if (!MoveBracketsToSucc) {
2602 MoveBracketsToSucc = &SuccBI;
2604 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
2606 }
else if (SuccBI.Incoming->merge(*Brackets)) {
2607 SuccBI.Dirty =
true;
2612 if (MoveBracketsToSucc)
2613 MoveBracketsToSucc->Incoming = std::move(Brackets);
2618 if (
ST->hasScalarStores()) {
2620 bool HaveScalarStores =
false;
2624 if (!HaveScalarStores &&
TII->isScalarStore(
MI))
2625 HaveScalarStores =
true;
2627 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
2628 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
2633 if (HaveScalarStores) {
2643 bool SeenDCacheWB =
false;
2647 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
2648 SeenDCacheWB =
true;
2649 else if (
TII->isScalarStore(*
I))
2650 SeenDCacheWB =
false;
2653 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
2654 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
2667 if (
ST->requiresNopBeforeDeallocVGPRs()) {
2672 TII->get(AMDGPU::S_SENDMSG))
2676 ReleaseVGPRInsts.clear();
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Analysis containing CSE Info
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool isCacheInvOrWBInst(MachineInstr &Inst)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static std::optional< InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
static bool readsVCCZ(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isEntryFunction() const
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptNone() const
Do not optimize this function (-O0).
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Represents a single loop in the control flow graph.
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Instructions::iterator instr_iterator
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
unsigned getNumOperands() const
Retuns the total number of operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isEXP(const MachineInstr &MI)
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
static bool isVIMAGE(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isVSAMPLE(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned getStorecntBitMask(const IsaVersion &Version)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getKmcntBitMask(const IsaVersion &Version)
unsigned getVmcntBitMask(const IsaVersion &Version)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getBvhcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
static unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt, unsigned Dscnt)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getLoadcntBitMask(const IsaVersion &Version)
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt, unsigned Dscnt)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Represents the counter values to wait for in an s_waitcnt instruction.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static constexpr bool is_iterable