42#define DEBUG_TYPE "si-insert-waitcnts"
45 "Force emit s_waitcnt expcnt(0) instrs");
47 "Force emit s_waitcnt lgkmcnt(0) instrs");
49 "Force emit s_waitcnt vmcnt(0) instrs");
52 "amdgpu-waitcnt-forcezero",
53 cl::desc(
"Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
67 SAMPLE_CNT = NUM_NORMAL_INST_CNTS,
70 NUM_EXTENDED_INST_CNTS,
71 NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
85auto inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS) {
86 return enum_seq(LOAD_CNT, MaxCounter);
89using RegInterval = std::pair<int, int>;
91struct HardwareLimits {
96 unsigned SamplecntMax;
101struct RegisterEncoding {
111 VMEM_SAMPLER_READ_ACCESS,
112 VMEM_BVH_READ_ACCESS,
114 SCRATCH_WRITE_ACCESS,
134enum RegisterMapping {
135 SQ_MAX_PGM_VGPRS = 512,
137 SQ_MAX_PGM_SGPRS = 256,
145 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS,
166static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
177static bool isNormalMode(InstCounterType MaxCounter) {
178 return MaxCounter == NUM_NORMAL_INST_CNTS;
183 assert(updateVMCntOnly(Inst));
186 return VMEM_NOSAMPLER;
193 return BaseInfo->
BVH ? VMEM_BVH
207 return Wait.StoreCnt;
209 return Wait.SampleCnt;
220 unsigned &WC = getCounterRef(
Wait,
T);
221 WC = std::min(WC, Count);
225 getCounterRef(
Wait,
T) = ~0
u;
229 return getCounterRef(
Wait,
T);
233InstCounterType eventCounter(
const unsigned *masks, WaitEventType E) {
234 for (
auto T : inst_counter_types()) {
235 if (masks[
T] & (1 << E))
249class WaitcntBrackets {
251 WaitcntBrackets(
const GCNSubtarget *SubTarget, InstCounterType MaxCounter,
252 HardwareLimits Limits, RegisterEncoding Encoding,
253 const unsigned *WaitEventMaskForInst,
254 InstCounterType SmemAccessCounter)
255 :
ST(SubTarget), MaxCounter(MaxCounter), Limits(Limits),
256 Encoding(Encoding), WaitEventMaskForInst(WaitEventMaskForInst),
257 SmemAccessCounter(SmemAccessCounter) {}
259 unsigned getWaitCountMax(InstCounterType
T)
const {
262 return Limits.LoadcntMax;
264 return Limits.DscntMax;
266 return Limits.ExpcntMax;
268 return Limits.StorecntMax;
270 return Limits.SamplecntMax;
272 return Limits.BvhcntMax;
274 return Limits.KmcntMax;
281 unsigned getScoreLB(InstCounterType
T)
const {
286 unsigned getScoreUB(InstCounterType
T)
const {
291 unsigned getScoreRange(InstCounterType
T)
const {
292 return getScoreUB(
T) - getScoreLB(
T);
295 unsigned getRegScore(
int GprNo, InstCounterType
T)
const {
296 if (GprNo < NUM_ALL_VGPRS) {
297 return VgprScores[
T][GprNo];
299 assert(
T == SmemAccessCounter);
300 return SgprScores[GprNo - NUM_ALL_VGPRS];
309 bool counterOutOfOrder(InstCounterType
T)
const;
311 void simplifyWaitcnt(InstCounterType
T,
unsigned &Count)
const;
314 void applyWaitcnt(InstCounterType
T,
unsigned Count);
319 unsigned hasPendingEvent()
const {
return PendingEvents; }
320 unsigned hasPendingEvent(WaitEventType E)
const {
321 return PendingEvents & (1 << E);
323 unsigned hasPendingEvent(InstCounterType
T)
const {
324 unsigned HasPending = PendingEvents & WaitEventMaskForInst[
T];
325 assert((HasPending != 0) == (getScoreRange(
T) != 0));
329 bool hasMixedPendingEvents(InstCounterType
T)
const {
330 unsigned Events = hasPendingEvent(
T);
332 return Events & (Events - 1);
335 bool hasPendingFlat()
const {
336 return ((LastFlat[DS_CNT] > ScoreLBs[DS_CNT] &&
337 LastFlat[DS_CNT] <= ScoreUBs[DS_CNT]) ||
338 (LastFlat[LOAD_CNT] > ScoreLBs[LOAD_CNT] &&
339 LastFlat[LOAD_CNT] <= ScoreUBs[LOAD_CNT]));
342 void setPendingFlat() {
343 LastFlat[LOAD_CNT] = ScoreUBs[LOAD_CNT];
344 LastFlat[DS_CNT] = ScoreUBs[DS_CNT];
349 bool hasOtherPendingVmemTypes(
int GprNo, VmemType V)
const {
350 assert(GprNo < NUM_ALL_VGPRS);
351 return VgprVmemTypes[GprNo] & ~(1 <<
V);
354 void clearVgprVmemTypes(
int GprNo) {
355 assert(GprNo < NUM_ALL_VGPRS);
356 VgprVmemTypes[GprNo] = 0;
359 void setStateOnFunctionEntryOrReturn() {
360 setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getWaitCountMax(STORE_CNT));
361 PendingEvents |= WaitEventMaskForInst[STORE_CNT];
378 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
379 unsigned OtherScore);
381 void setScoreLB(InstCounterType
T,
unsigned Val) {
386 void setScoreUB(InstCounterType
T,
unsigned Val) {
393 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
397 void setRegScore(
int GprNo, InstCounterType
T,
unsigned Val) {
398 if (GprNo < NUM_ALL_VGPRS) {
399 VgprUB = std::max(VgprUB, GprNo);
400 VgprScores[
T][GprNo] = Val;
402 assert(
T == SmemAccessCounter);
403 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
404 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
410 unsigned OpNo,
unsigned Val);
413 InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
414 HardwareLimits Limits = {};
415 RegisterEncoding Encoding = {};
416 const unsigned *WaitEventMaskForInst;
417 InstCounterType SmemAccessCounter;
418 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
419 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
420 unsigned PendingEvents = 0;
422 unsigned LastFlat[NUM_INST_CNTS] = {0};
427 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
430 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
433 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
445class WaitcntGenerator {
450 InstCounterType MaxCounter;
454 WaitcntGenerator() =
default;
455 WaitcntGenerator(
const MachineFunction &MF, InstCounterType MaxCounter)
463 bool isOptNone()
const {
return OptNone; }
477 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
492 virtual const unsigned *getWaitEventMask()
const = 0;
496 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const = 0;
498 virtual ~WaitcntGenerator() =
default;
501 static constexpr unsigned
502 eventMask(std::initializer_list<WaitEventType> Events) {
504 for (
auto &E : Events)
511class WaitcntGeneratorPreGFX12 :
public WaitcntGenerator {
513 WaitcntGeneratorPreGFX12() =
default;
515 : WaitcntGenerator(MF, NUM_NORMAL_INST_CNTS) {}
518 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
526 const unsigned *getWaitEventMask()
const override {
529 static const unsigned WaitEventMaskForInstPreGFX12[NUM_INST_CNTS] = {
530 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS,
531 VMEM_BVH_READ_ACCESS}),
532 eventMask({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
533 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
534 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
535 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
540 return WaitEventMaskForInstPreGFX12;
546class WaitcntGeneratorGFX12Plus :
public WaitcntGenerator {
548 WaitcntGeneratorGFX12Plus() =
default;
550 InstCounterType MaxCounter)
551 : WaitcntGenerator(MF, MaxCounter) {}
554 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
562 const unsigned *getWaitEventMask()
const override {
565 static const unsigned WaitEventMaskForInstGFX12Plus[NUM_INST_CNTS] = {
566 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS}),
567 eventMask({LDS_ACCESS, GDS_ACCESS}),
568 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
569 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
570 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
571 eventMask({VMEM_SAMPLER_READ_ACCESS}),
572 eventMask({VMEM_BVH_READ_ACCESS}),
573 eventMask({SMEM_ACCESS, SQ_MESSAGE})};
575 return WaitEventMaskForInstGFX12Plus;
595 std::unique_ptr<WaitcntBrackets>
Incoming;
599 InstCounterType SmemAccessCounter;
605 bool ForceEmitZeroWaitcnts;
606 bool ForceEmitWaitcnt[NUM_INST_CNTS];
611 WaitcntGeneratorPreGFX12 WCGPreGFX12;
612 WaitcntGeneratorGFX12Plus WCGGFX12Plus;
614 WaitcntGenerator *WCG =
nullptr;
620 InstCounterType MaxCounter = NUM_NORMAL_INST_CNTS;
626 (void)ForceExpCounter;
627 (void)ForceLgkmCounter;
628 (void)ForceVMCounter;
631 bool shouldFlushVmCnt(
MachineLoop *
ML, WaitcntBrackets &Brackets);
633 WaitcntBrackets &ScoreBrackets);
638 return "SI insert wait instructions";
650 bool isForceEmitWaitcnt()
const {
651 for (
auto T : inst_counter_types())
652 if (ForceEmitWaitcnt[
T])
657 void setForceEmitWaitcnt() {
663 ForceEmitWaitcnt[
EXP_CNT] =
true;
665 ForceEmitWaitcnt[
EXP_CNT] =
false;
670 ForceEmitWaitcnt[DS_CNT] =
true;
671 ForceEmitWaitcnt[KM_CNT] =
true;
673 ForceEmitWaitcnt[DS_CNT] =
false;
674 ForceEmitWaitcnt[KM_CNT] =
false;
679 ForceEmitWaitcnt[LOAD_CNT] =
true;
680 ForceEmitWaitcnt[SAMPLE_CNT] =
true;
681 ForceEmitWaitcnt[BVH_CNT] =
true;
683 ForceEmitWaitcnt[LOAD_CNT] =
false;
684 ForceEmitWaitcnt[SAMPLE_CNT] =
false;
685 ForceEmitWaitcnt[BVH_CNT] =
false;
692 WaitEventType getVmemWaitEventType(
const MachineInstr &Inst)
const {
694 static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
695 VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
706 return SCRATCH_WRITE_ACCESS;
707 return VMEM_WRITE_ACCESS;
710 return VMEM_READ_ACCESS;
711 return VmemReadMapping[getVmemType(Inst)];
718 WaitcntBrackets &ScoreBrackets,
726 WaitcntBrackets *ScoreBrackets);
728 WaitcntBrackets &ScoreBrackets);
733RegInterval WaitcntBrackets::getRegInterval(
const MachineInstr *
MI,
736 unsigned OpNo)
const {
738 if (!
TRI->isInAllocatableClass(
Op.getReg()))
750 if (
TRI->isVectorRegister(*
MRI,
Op.getReg())) {
751 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
754 Result.first += AGPR_OFFSET;
756 }
else if (
TRI->isSGPRReg(*
MRI,
Op.getReg())) {
757 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
758 Result.first =
Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
760 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
768 unsigned Size =
TRI->getRegSizeInBits(*RC);
780 assert(
TRI->isVectorRegister(*
MRI,
MI->getOperand(OpNo).getReg()));
782 setRegScore(RegNo, EXP_CNT, Val);
790 InstCounterType
T = eventCounter(WaitEventMaskForInst, E);
792 unsigned UB = getScoreUB(
T);
793 unsigned CurrScore = UB + 1;
799 PendingEvents |= 1 << E;
800 setScoreUB(
T, CurrScore);
810 if (AddrOpIdx != -1) {
811 setExpScore(&Inst,
TII,
TRI,
MRI, AddrOpIdx, CurrScore);
824 AMDGPU::OpName::data1),
829 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
830 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
833 if (
Op.isReg() && !
Op.isDef() &&
834 TRI->isVectorRegister(*
MRI,
Op.getReg())) {
835 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
839 }
else if (
TII->isFLAT(Inst)) {
851 }
else if (
TII->isMIMG(Inst)) {
853 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
860 }
else if (
TII->isMTBUF(Inst)) {
862 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
864 }
else if (
TII->isMUBUF(Inst)) {
866 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
873 }
else if (
TII->isLDSDIR(Inst)) {
880 if (
TII->isEXP(Inst)) {
899 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
907 if (!
Op.isReg() || !
Op.isDef())
910 if (
T == LOAD_CNT ||
T == SAMPLE_CNT ||
T == BVH_CNT) {
911 if (
Interval.first >= NUM_ALL_VGPRS)
913 if (updateVMCntOnly(Inst)) {
918 VmemType
V = getVmemType(Inst);
920 VgprVmemTypes[RegNo] |= 1 <<
V;
924 setRegScore(RegNo,
T, CurrScore);
928 (
TII->isDS(Inst) ||
TII->mayWriteLDSThroughDMA(Inst))) {
933 if (!
MemOp->isStore() ||
938 auto AAI =
MemOp->getAAInfo();
946 if (!AAI || !AAI.Scope)
948 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E && !Slot; ++
I) {
949 for (
const auto *
MemOp : LDSDMAStores[
I]->memoperands()) {
950 if (
MemOp->isStore() && AAI ==
MemOp->getAAInfo()) {
956 if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
958 LDSDMAStores.push_back(&Inst);
959 Slot = LDSDMAStores.size();
962 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot,
T, CurrScore);
964 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS,
T, CurrScore);
971 for (
auto T : inst_counter_types(MaxCounter)) {
972 unsigned SR = getScoreRange(
T);
976 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"LOAD" :
"VM") <<
"_CNT("
980 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"DS" :
"LGKM") <<
"_CNT("
984 OS <<
" EXP_CNT(" << SR <<
"): ";
987 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"STORE" :
"VS") <<
"_CNT("
991 OS <<
" SAMPLE_CNT(" << SR <<
"): ";
994 OS <<
" BVH_CNT(" << SR <<
"): ";
997 OS <<
" KM_CNT(" << SR <<
"): ";
1000 OS <<
" UNKNOWN(" << SR <<
"): ";
1006 unsigned LB = getScoreLB(
T);
1008 for (
int J = 0; J <= VgprUB; J++) {
1009 unsigned RegScore = getRegScore(J,
T);
1012 unsigned RelScore = RegScore - LB - 1;
1013 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
1014 OS << RelScore <<
":v" << J <<
" ";
1016 OS << RelScore <<
":ds ";
1020 if (
T == SmemAccessCounter) {
1021 for (
int J = 0; J <= SgprUB; J++) {
1022 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS,
T);
1025 unsigned RelScore = RegScore - LB - 1;
1026 OS << RelScore <<
":s" << J <<
" ";
1038 simplifyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1039 simplifyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1040 simplifyWaitcnt(DS_CNT,
Wait.DsCnt);
1041 simplifyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1042 simplifyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1043 simplifyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1044 simplifyWaitcnt(KM_CNT,
Wait.KmCnt);
1047void WaitcntBrackets::simplifyWaitcnt(InstCounterType
T,
1048 unsigned &Count)
const {
1052 if (Count >= getScoreRange(
T))
1056void WaitcntBrackets::determineWait(InstCounterType
T,
int RegNo,
1058 unsigned ScoreToWait = getRegScore(RegNo,
T);
1062 const unsigned LB = getScoreLB(
T);
1063 const unsigned UB = getScoreUB(
T);
1064 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1065 if ((
T == LOAD_CNT ||
T == DS_CNT) && hasPendingFlat() &&
1066 !
ST->hasFlatLgkmVMemCountInOrder()) {
1070 addWait(
Wait,
T, 0);
1071 }
else if (counterOutOfOrder(
T)) {
1075 addWait(
Wait,
T, 0);
1079 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(
T) - 1);
1080 addWait(
Wait,
T, NeededWait);
1086 applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1087 applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1088 applyWaitcnt(DS_CNT,
Wait.DsCnt);
1089 applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1090 applyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1091 applyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1092 applyWaitcnt(KM_CNT,
Wait.KmCnt);
1095void WaitcntBrackets::applyWaitcnt(InstCounterType
T,
unsigned Count) {
1096 const unsigned UB = getScoreUB(
T);
1100 if (counterOutOfOrder(
T))
1102 setScoreLB(
T, std::max(getScoreLB(
T), UB - Count));
1105 PendingEvents &= ~WaitEventMaskForInst[
T];
1111bool WaitcntBrackets::counterOutOfOrder(InstCounterType
T)
const {
1113 if (
T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
1115 return hasMixedPendingEvents(
T);
1125char SIInsertWaitcnts::
ID = 0;
1130 return new SIInsertWaitcnts();
1140 if (NewEnc == MO.
getImm())
1151 case AMDGPU::S_WAIT_LOADCNT:
1153 case AMDGPU::S_WAIT_EXPCNT:
1155 case AMDGPU::S_WAIT_STORECNT:
1157 case AMDGPU::S_WAIT_SAMPLECNT:
1159 case AMDGPU::S_WAIT_BVHCNT:
1161 case AMDGPU::S_WAIT_DSCNT:
1163 case AMDGPU::S_WAIT_KMCNT:
1170bool WaitcntGenerator::promoteSoftWaitCnt(
MachineInstr *Waitcnt)
const {
1184bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1185 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1188 assert(isNormalMode(MaxCounter));
1196 if (
II.isMetaInstruction())
1200 bool TrySimplify = Opcode !=
II.getOpcode() && !OptNone;
1204 if (Opcode == AMDGPU::S_WAITCNT) {
1205 unsigned IEnc =
II.getOperand(0).getImm();
1208 ScoreBrackets.simplifyWaitcnt(OldWait);
1212 if (WaitcntInstr || (!
Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1213 II.eraseFromParent();
1218 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1219 assert(
II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1222 TII->getNamedOperand(
II, AMDGPU::OpName::simm16)->getImm();
1224 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt);
1225 Wait.StoreCnt = std::min(
Wait.StoreCnt, OldVSCnt);
1227 if (WaitcntVsCntInstr || (!
Wait.hasWaitStoreCnt() && TrySimplify)) {
1228 II.eraseFromParent();
1231 WaitcntVsCntInstr = &
II;
1238 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1240 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1241 ScoreBrackets.applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1242 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1249 <<
"applyPreexistingWaitcnt\n"
1250 <<
"New Instr at block end: " << *WaitcntInstr <<
'\n'
1251 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1252 <<
"Old Instr: " << *It
1253 <<
"New Instr: " << *WaitcntInstr <<
'\n');
1256 if (WaitcntVsCntInstr) {
1258 AMDGPU::OpName::simm16,
Wait.StoreCnt);
1259 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1261 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1262 Wait.StoreCnt = ~0
u;
1265 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1266 <<
"New Instr at block end: " << *WaitcntVsCntInstr
1268 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1269 <<
"Old Instr: " << *It
1270 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
1278bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1282 assert(isNormalMode(MaxCounter));
1289 if (
Wait.hasWaitExceptStoreCnt()) {
1291 [[maybe_unused]]
auto SWaitInst =
1296 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1297 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1300 if (
Wait.hasWaitStoreCnt()) {
1303 [[maybe_unused]]
auto SWaitInst =
1310 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1311 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1318WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1323WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1331bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1332 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1335 assert(!isNormalMode(MaxCounter));
1344 if (
II.isMetaInstruction())
1353 bool TrySimplify = Opcode !=
II.getOpcode() && !OptNone;
1357 if (Opcode == AMDGPU::S_WAITCNT)
1360 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1362 TII->getNamedOperand(
II, AMDGPU::OpName::simm16)->getImm();
1365 ScoreBrackets.simplifyWaitcnt(OldWait);
1367 UpdatableInstr = &CombinedLoadDsCntInstr;
1368 }
else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1370 TII->getNamedOperand(
II, AMDGPU::OpName::simm16)->getImm();
1373 ScoreBrackets.simplifyWaitcnt(OldWait);
1375 UpdatableInstr = &CombinedStoreDsCntInstr;
1380 TII->getNamedOperand(
II, AMDGPU::OpName::simm16)->getImm();
1382 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt);
1383 addWait(
Wait, CT.value(), OldCnt);
1384 UpdatableInstr = &WaitInstrs[CT.value()];
1388 if (!*UpdatableInstr) {
1389 *UpdatableInstr = &
II;
1391 II.eraseFromParent();
1396 if (CombinedLoadDsCntInstr) {
1404 if (
Wait.LoadCnt != ~0u &&
Wait.DsCnt != ~0u) {
1407 AMDGPU::OpName::simm16, NewEnc);
1408 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
1409 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1410 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1415 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1416 <<
"New Instr at block end: "
1417 << *CombinedLoadDsCntInstr <<
'\n'
1418 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1419 <<
"Old Instr: " << *It <<
"New Instr: "
1420 << *CombinedLoadDsCntInstr <<
'\n');
1427 if (CombinedStoreDsCntInstr) {
1429 if (
Wait.StoreCnt != ~0u &&
Wait.DsCnt != ~0u) {
1432 AMDGPU::OpName::simm16, NewEnc);
1433 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
1434 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1435 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1436 Wait.StoreCnt = ~0
u;
1440 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1441 <<
"New Instr at block end: "
1442 << *CombinedStoreDsCntInstr <<
'\n'
1443 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1444 <<
"Old Instr: " << *It <<
"New Instr: "
1445 << *CombinedStoreDsCntInstr <<
'\n');
1458 if (
Wait.DsCnt != ~0u) {
1467 if (
Wait.LoadCnt != ~0u) {
1468 WaitsToErase.
push_back(&WaitInstrs[LOAD_CNT]);
1469 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1470 }
else if (
Wait.StoreCnt != ~0u) {
1471 WaitsToErase.
push_back(&WaitInstrs[STORE_CNT]);
1472 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1479 (*WI)->eraseFromParent();
1485 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1486 if (!WaitInstrs[CT])
1489 unsigned NewCnt = getWait(
Wait, CT);
1490 if (NewCnt != ~0u) {
1492 AMDGPU::OpName::simm16, NewCnt);
1493 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
1495 ScoreBrackets.applyWaitcnt(CT, NewCnt);
1496 setNoWait(
Wait, CT);
1499 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1500 <<
"New Instr at block end: " << *WaitInstrs[CT]
1502 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1503 <<
"Old Instr: " << *It
1504 <<
"New Instr: " << *WaitInstrs[CT] <<
'\n');
1515bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
1519 assert(!isNormalMode(MaxCounter));
1525 if (
Wait.DsCnt != ~0u) {
1528 if (
Wait.LoadCnt != ~0u) {
1536 }
else if (
Wait.StoreCnt != ~0u) {
1543 Wait.StoreCnt = ~0
u;
1551 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1552 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1559 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1560 unsigned Count = getWait(
Wait, CT);
1564 [[maybe_unused]]
auto SWaitInst =
1571 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1572 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1579 unsigned Opc =
MI.getOpcode();
1580 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
1581 !
MI.getOperand(1).isUndef();
1611bool SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &
MI,
1612 WaitcntBrackets &ScoreBrackets,
1615 setForceEmitWaitcnt();
1617 if (
MI.isMetaInstruction())
1626 if (
MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1627 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1628 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1629 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1630 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1637 if (
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1638 MI.getOpcode() == AMDGPU::SI_RETURN ||
1639 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1641 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
false));
1649 else if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1650 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1652 ScoreBrackets.getScoreRange(STORE_CNT) != 0 &&
1653 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1657 else if ((
MI.getOpcode() == AMDGPU::S_SENDMSG ||
1658 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1659 ST->hasLegacyGeometry() &&
1670 if (
MI.modifiesRegister(AMDGPU::EXEC,
TRI)) {
1673 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1674 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1675 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1676 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1690 if (
MI.getOperand(CallAddrOpIdx).isReg()) {
1691 RegInterval CallAddrOpInterval =
1692 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, CallAddrOpIdx);
1694 for (
int RegNo = CallAddrOpInterval.first;
1695 RegNo < CallAddrOpInterval.second; ++RegNo)
1696 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1700 if (RtnAddrOpIdx != -1) {
1701 RegInterval RtnAddrOpInterval =
1702 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, RtnAddrOpIdx);
1704 for (
int RegNo = RtnAddrOpInterval.first;
1705 RegNo < RtnAddrOpInterval.second; ++RegNo)
1706 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1725 const Value *
Ptr = Memop->getValue();
1726 if (Memop->isStore() && SLoadAddresses.
count(
Ptr)) {
1727 addWait(
Wait, SmemAccessCounter, 0);
1731 unsigned AS = Memop->getAddrSpace();
1735 if (
TII->mayWriteLDSThroughDMA(
MI))
1739 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1740 bool FoundAliasingStore =
false;
1747 if (
Ptr && Memop->getAAInfo() && Memop->getAAInfo().Scope) {
1748 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
1749 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E; ++
I) {
1750 if (
MI.mayAlias(AA, *LDSDMAStores[
I],
true)) {
1751 FoundAliasingStore =
true;
1752 ScoreBrackets.determineWait(LOAD_CNT, RegNo +
I + 1,
Wait);
1756 if (!FoundAliasingStore)
1757 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1758 if (Memop->isStore()) {
1759 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1764 for (
unsigned I = 0, E =
MI.getNumOperands();
I != E; ++
I) {
1770 if (
Op.isTied() &&
Op.isUse() &&
TII->doesNotReadTiedSource(
MI))
1775 const bool IsVGPR =
TRI->isVectorRegister(*
MRI,
Op.getReg());
1782 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
1783 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1785 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1786 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo,
Wait);
1787 ScoreBrackets.determineWait(BVH_CNT, RegNo,
Wait);
1788 ScoreBrackets.clearVgprVmemTypes(RegNo);
1790 if (
Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1791 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1793 ScoreBrackets.determineWait(DS_CNT, RegNo,
Wait);
1795 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1806 if (
TII->isBarrierStart(
MI.getOpcode()) &&
1807 !
ST->hasAutoWaitcntBeforeBarrier() && !
ST->supportsBackOffBarrier()) {
1808 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
true));
1815 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1821 ScoreBrackets.simplifyWaitcnt(
Wait);
1823 if (ForceEmitZeroWaitcnts)
1824 Wait = WCG->getAllZeroWaitcnt(
false);
1826 if (ForceEmitWaitcnt[LOAD_CNT])
1828 if (ForceEmitWaitcnt[EXP_CNT])
1830 if (ForceEmitWaitcnt[DS_CNT])
1832 if (ForceEmitWaitcnt[SAMPLE_CNT])
1834 if (ForceEmitWaitcnt[BVH_CNT])
1836 if (ForceEmitWaitcnt[KM_CNT])
1840 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
1842 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
1844 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
1848 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
1855 WaitcntBrackets &ScoreBrackets,
1859 if (OldWaitcntInstr)
1863 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
1867 ScoreBrackets.applyWaitcnt(
Wait);
1870 if (
Wait.ExpCnt != ~0u && It !=
Block.instr_end() &&
1873 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1881 <<
"Update Instr: " << *It);
1884 if (WCG->createNewWaitcnt(
Block, It,
Wait))
1893bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(
const MachineInstr &
MI)
const {
1901 if (
MI.memoperands_empty())
1910 unsigned AS = Memop->getAddrSpace();
1921bool SIInsertWaitcnts::mayAccessLDSThroughFlat(
const MachineInstr &
MI)
const {
1925 if (!
TII->usesLGKM_CNT(
MI))
1929 if (
ST->isTgSplitEnabled())
1934 if (
MI.memoperands_empty())
1939 unsigned AS = Memop->getAddrSpace();
1949bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
1954 if (
TII->isFLATScratch(
MI))
1958 if (
TII->isFLATGlobal(
MI))
1963 if (
MI.memoperands_empty())
1968 unsigned AS = Memop->getAddrSpace();
1969 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
1975 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1976 Opc == AMDGPU::GLOBAL_WBINV;
1979void SIInsertWaitcnts::updateEventWaitcntAfter(
MachineInstr &Inst,
1980 WaitcntBrackets *ScoreBrackets) {
1986 if (
TII->isDS(Inst) &&
TII->usesLGKM_CNT(Inst)) {
1988 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
1989 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_ACCESS, Inst);
1990 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_GPR_LOCK, Inst);
1992 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
1994 }
else if (
TII->isFLAT(Inst)) {
2001 int FlatASCount = 0;
2003 if (mayAccessVMEMThroughFlat(Inst)) {
2005 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2009 if (mayAccessLDSThroughFlat(Inst)) {
2011 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2020 if (FlatASCount > 1)
2021 ScoreBrackets->setPendingFlat();
2024 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2027 if (
ST->vmemWriteNeedsExpWaitcnt() &&
2029 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, VMW_GPR_LOCK, Inst);
2031 }
else if (
TII->isSMRD(Inst)) {
2032 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2033 }
else if (Inst.
isCall()) {
2036 ScoreBrackets->applyWaitcnt(
2037 WCG->getAllZeroWaitcnt(
false));
2038 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2044 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_LDS_ACCESS, Inst);
2045 }
else if (
TII->isVINTERP(Inst)) {
2046 int64_t
Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2047 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
2049 unsigned Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
2051 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_PARAM_ACCESS, Inst);
2053 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_POS_ACCESS, Inst);
2055 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_GPR_LOCK, Inst);
2058 case AMDGPU::S_SENDMSG:
2059 case AMDGPU::S_SENDMSG_RTN_B32:
2060 case AMDGPU::S_SENDMSG_RTN_B64:
2061 case AMDGPU::S_SENDMSGHALT:
2062 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SQ_MESSAGE, Inst);
2064 case AMDGPU::S_MEMTIME:
2065 case AMDGPU::S_MEMREALTIME:
2066 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
2067 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
2068 case AMDGPU::S_BARRIER_LEAVE:
2069 case AMDGPU::S_GET_BARRIER_STATE_M0:
2070 case AMDGPU::S_GET_BARRIER_STATE_IMM:
2071 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2077bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
2078 unsigned OtherScore) {
2079 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
2080 unsigned OtherShifted =
2081 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
2082 Score = std::max(MyShifted, OtherShifted);
2083 return OtherShifted > MyShifted;
2091bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
2092 bool StrictDom =
false;
2094 VgprUB = std::max(VgprUB,
Other.VgprUB);
2095 SgprUB = std::max(SgprUB,
Other.SgprUB);
2097 for (
auto T : inst_counter_types(MaxCounter)) {
2099 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[
T];
2100 const unsigned OtherEvents =
Other.PendingEvents & WaitEventMaskForInst[
T];
2101 if (OtherEvents & ~OldEvents)
2103 PendingEvents |= OtherEvents;
2106 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
2107 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
2108 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
2109 if (NewUB < ScoreLBs[
T])
2113 M.OldLB = ScoreLBs[
T];
2114 M.OtherLB =
Other.ScoreLBs[
T];
2115 M.MyShift = NewUB - ScoreUBs[
T];
2116 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
2118 ScoreUBs[
T] = NewUB;
2120 StrictDom |= mergeScore(M, LastFlat[
T],
Other.LastFlat[
T]);
2122 for (
int J = 0; J <= VgprUB; J++)
2123 StrictDom |= mergeScore(M, VgprScores[
T][J],
Other.VgprScores[
T][J]);
2125 if (
T == SmemAccessCounter) {
2126 for (
int J = 0; J <= SgprUB; J++)
2127 StrictDom |= mergeScore(M, SgprScores[J],
Other.SgprScores[J]);
2131 for (
int J = 0; J <= VgprUB; J++) {
2132 unsigned char NewVmemTypes = VgprVmemTypes[J] |
Other.VgprVmemTypes[J];
2133 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
2134 VgprVmemTypes[J] = NewVmemTypes;
2142 return Opcode == AMDGPU::S_WAITCNT ||
2145 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2146 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2153 WaitcntBrackets &ScoreBrackets) {
2157 dbgs() <<
"*** Block" <<
Block.getNumber() <<
" ***";
2158 ScoreBrackets.dump();
2164 bool VCCZCorrect =
true;
2165 if (
ST->hasReadVCCZBug()) {
2168 VCCZCorrect =
false;
2169 }
else if (!
ST->partialVCCWritesUpdateVCCZ()) {
2172 VCCZCorrect =
false;
2179 E =
Block.instr_end();
2186 if (!OldWaitcntInstr)
2187 OldWaitcntInstr = &Inst;
2192 bool FlushVmCnt =
Block.getFirstTerminator() == Inst &&
2193 isPreheaderToFlush(
Block, ScoreBrackets);
2196 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
2198 OldWaitcntInstr =
nullptr;
2201 bool RestoreVCCZ = !VCCZCorrect &&
readsVCCZ(Inst);
2204 if (
ST->hasReadVCCZBug() || !
ST->partialVCCWritesUpdateVCCZ()) {
2208 if (!
ST->partialVCCWritesUpdateVCCZ())
2209 VCCZCorrect =
false;
2218 if (
ST->hasReadVCCZBug() &&
2219 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
2222 VCCZCorrect =
false;
2230 if (
TII->isSMRD(Inst)) {
2234 if (!Memop->isInvariant()) {
2235 const Value *
Ptr = Memop->getValue();
2239 if (
ST->hasReadVCCZBug()) {
2241 VCCZCorrect =
false;
2245 updateEventWaitcntAfter(Inst, &ScoreBrackets);
2250 ScoreBrackets.simplifyWaitcnt(
Wait);
2252 ScoreBrackets,
nullptr);
2257 ScoreBrackets.dump();
2267 TII->get(
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
2280 if (
Block.getFirstTerminator() ==
Block.end() &&
2281 isPreheaderToFlush(
Block, ScoreBrackets)) {
2282 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
2284 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
2286 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
2300 WaitcntBrackets &ScoreBrackets) {
2301 auto [Iterator, IsInserted] = PreheadersToFlush.
try_emplace(&
MBB,
false);
2303 return Iterator->second;
2314 shouldFlushVmCnt(
Loop, ScoreBrackets)) {
2315 Iterator->second =
true;
2322bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
2336 WaitcntBrackets &Brackets) {
2337 bool HasVMemLoad =
false;
2338 bool HasVMemStore =
false;
2339 bool UsesVgprLoadedOutside =
false;
2345 if (isVMEMOrFlatVMEM(
MI)) {
2349 HasVMemStore =
true;
2351 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++) {
2353 if (!
Op.isReg() || !
TRI->isVectorRegister(*
MRI,
Op.getReg()))
2366 if (Brackets.getRegScore(RegNo, LOAD_CNT) >
2367 Brackets.getScoreLB(LOAD_CNT) ||
2368 Brackets.getRegScore(RegNo, SAMPLE_CNT) >
2369 Brackets.getScoreLB(SAMPLE_CNT) ||
2370 Brackets.getRegScore(RegNo, BVH_CNT) >
2371 Brackets.getScoreLB(BVH_CNT)) {
2372 UsesVgprLoadedOutside =
true;
2378 else if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad() &&
Op.isDef())
2389 if (!
ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
2391 return HasVMemLoad && UsesVgprLoadedOutside;
2396 TII =
ST->getInstrInfo();
2397 TRI = &
TII->getRegisterInfo();
2400 MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
2401 PDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
2402 if (
auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
2403 AA = &AAR->getAAResults();
2407 if (
ST->hasExtendedWaitCounts()) {
2408 MaxCounter = NUM_EXTENDED_INST_CNTS;
2409 WCGGFX12Plus = WaitcntGeneratorGFX12Plus(MF, MaxCounter);
2410 WCG = &WCGGFX12Plus;
2412 MaxCounter = NUM_NORMAL_INST_CNTS;
2413 WCGPreGFX12 = WaitcntGeneratorPreGFX12(MF);
2418 for (
auto T : inst_counter_types())
2419 ForceEmitWaitcnt[
T] =
false;
2421 const unsigned *WaitEventMaskForInst = WCG->getWaitEventMask();
2423 SmemAccessCounter = eventCounter(WaitEventMaskForInst, SMEM_ACCESS);
2425 HardwareLimits Limits = {};
2426 if (
ST->hasExtendedWaitCounts()) {
2439 unsigned NumVGPRsMax =
ST->getAddressableNumVGPRs();
2440 unsigned NumSGPRsMax =
ST->getAddressableNumSGPRs();
2441 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
2442 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
2444 RegisterEncoding Encoding = {};
2447 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
2450 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
2466 I != E && (
I->isPHI() ||
I->isMetaInstruction()); ++
I)
2469 if (
ST->hasExtendedWaitCounts()) {
2472 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
2473 if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
2477 TII->get(instrsForExtendedCounterTypes[CT]))
2484 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
2485 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2487 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
2488 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
2498 std::unique_ptr<WaitcntBrackets> Brackets;
2503 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
2506 BlockInfo &BI = BII->second;
2512 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
2514 *Brackets = *BI.Incoming;
2517 Brackets = std::make_unique<WaitcntBrackets>(
2518 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2521 *Brackets = WaitcntBrackets(ST, MaxCounter, Limits, Encoding,
2522 WaitEventMaskForInst, SmemAccessCounter);
2525 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
2528 if (Brackets->hasPendingEvent()) {
2529 BlockInfo *MoveBracketsToSucc =
nullptr;
2531 auto SuccBII = BlockInfos.
find(Succ);
2532 BlockInfo &SuccBI = SuccBII->second;
2533 if (!SuccBI.Incoming) {
2534 SuccBI.Dirty =
true;
2537 if (!MoveBracketsToSucc) {
2538 MoveBracketsToSucc = &SuccBI;
2540 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
2542 }
else if (SuccBI.Incoming->merge(*Brackets)) {
2543 SuccBI.Dirty =
true;
2548 if (MoveBracketsToSucc)
2549 MoveBracketsToSucc->Incoming = std::move(Brackets);
2554 if (
ST->hasScalarStores()) {
2556 bool HaveScalarStores =
false;
2560 if (!HaveScalarStores &&
TII->isScalarStore(
MI))
2561 HaveScalarStores =
true;
2563 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
2564 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
2569 if (HaveScalarStores) {
2579 bool SeenDCacheWB =
false;
2583 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
2584 SeenDCacheWB =
true;
2585 else if (
TII->isScalarStore(*
I))
2586 SeenDCacheWB =
false;
2589 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
2590 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
2603 if (
ST->requiresNopBeforeDeallocVGPRs()) {
2608 TII->get(AMDGPU::S_SENDMSG))
2612 ReleaseVGPRInsts.clear();
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Analysis containing CSE Info
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
static Function * getFunction(Constant *C)
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool isCacheInvOrWBInst(MachineInstr &Inst)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static std::optional< InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
static bool readsVCCZ(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isEntryFunction() const
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
FunctionPass class - This class is used to implement most global optimizations.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Instructions::iterator instr_iterator
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
unsigned getNumOperands() const
Retuns the total number of operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isEXP(const MachineInstr &MI)
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
static bool isVIMAGE(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isVSAMPLE(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned getStorecntBitMask(const IsaVersion &Version)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getKmcntBitMask(const IsaVersion &Version)
unsigned getVmcntBitMask(const IsaVersion &Version)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getBvhcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
static unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt, unsigned Dscnt)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getLoadcntBitMask(const IsaVersion &Version)
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt, unsigned Dscnt)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Represents the counter values to wait for in an s_waitcnt instruction.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static constexpr bool is_iterable