41#define DEBUG_TYPE "si-insert-waitcnts"
44 "Force emit s_waitcnt expcnt(0) instrs");
46 "Force emit s_waitcnt lgkmcnt(0) instrs");
48 "Force emit s_waitcnt vmcnt(0) instrs");
51 "amdgpu-waitcnt-forcezero",
52 cl::desc(
"Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
60enum InstCounterType { VM_CNT = 0, LGKM_CNT, EXP_CNT, VS_CNT, NUM_INST_CNTS };
70auto inst_counter_types() {
return enum_seq(VM_CNT, NUM_INST_CNTS); }
72using RegInterval = std::pair<int, int>;
74struct HardwareLimits {
81struct RegisterEncoding {
106static const unsigned WaitEventMaskForInst[NUM_INST_CNTS] = {
107 (1 << VMEM_ACCESS) | (1 << VMEM_READ_ACCESS),
108 (1 << SMEM_ACCESS) | (1 << LDS_ACCESS) | (1 << GDS_ACCESS) |
110 (1 << EXP_GPR_LOCK) | (1 << GDS_GPR_LOCK) | (1 << VMW_GPR_LOCK) |
111 (1 << EXP_PARAM_ACCESS) | (1 << EXP_POS_ACCESS) | (1 << EXP_LDS_ACCESS),
112 (1 << VMEM_WRITE_ACCESS) | (1 << SCRATCH_WRITE_ACCESS)};
120enum RegisterMapping {
121 SQ_MAX_PGM_VGPRS = 512,
123 SQ_MAX_PGM_SGPRS = 256,
126 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS,
149 assert(updateVMCntOnly(Inst));
151 return VMEM_NOSAMPLER;
155 return BaseInfo->
BVH ? VMEM_BVH
156 : BaseInfo->
Sampler ? VMEM_SAMPLER : VMEM_NOSAMPLER;
162 Wait.VmCnt = std::min(
Wait.VmCnt, Count);
165 Wait.ExpCnt = std::min(
Wait.ExpCnt, Count);
168 Wait.LgkmCnt = std::min(
Wait.LgkmCnt, Count);
171 Wait.VsCnt = std::min(
Wait.VsCnt, Count);
186class WaitcntBrackets {
188 WaitcntBrackets(
const GCNSubtarget *SubTarget, HardwareLimits Limits,
189 RegisterEncoding Encoding)
190 :
ST(SubTarget), Limits(Limits), Encoding(Encoding) {}
192 unsigned getWaitCountMax(InstCounterType
T)
const {
195 return Limits.VmcntMax;
197 return Limits.LgkmcntMax;
199 return Limits.ExpcntMax;
201 return Limits.VscntMax;
208 unsigned getScoreLB(InstCounterType
T)
const {
213 unsigned getScoreUB(InstCounterType
T)
const {
218 unsigned getScoreRange(InstCounterType
T)
const {
219 return getScoreUB(
T) - getScoreLB(
T);
223 InstCounterType eventCounter(WaitEventType
E)
const {
224 for (
auto T : inst_counter_types()) {
225 if (WaitEventMaskForInst[
T] & (1 <<
E))
231 unsigned getRegScore(
int GprNo, InstCounterType
T)
const {
232 if (GprNo < NUM_ALL_VGPRS) {
233 return VgprScores[
T][GprNo];
236 return SgprScores[GprNo - NUM_ALL_VGPRS];
245 bool counterOutOfOrder(InstCounterType
T)
const;
247 void simplifyWaitcnt(InstCounterType
T,
unsigned &Count)
const;
250 void applyWaitcnt(InstCounterType
T,
unsigned Count);
255 unsigned hasPendingEvent()
const {
return PendingEvents; }
256 unsigned hasPendingEvent(WaitEventType
E)
const {
257 return PendingEvents & (1 <<
E);
259 unsigned hasPendingEvent(InstCounterType
T)
const {
260 unsigned HasPending = PendingEvents & WaitEventMaskForInst[
T];
261 assert((HasPending != 0) == (getScoreRange(
T) != 0));
265 bool hasMixedPendingEvents(InstCounterType
T)
const {
266 unsigned Events = hasPendingEvent(
T);
268 return Events & (Events - 1);
271 bool hasPendingFlat()
const {
272 return ((LastFlat[LGKM_CNT] > ScoreLBs[LGKM_CNT] &&
273 LastFlat[LGKM_CNT] <= ScoreUBs[LGKM_CNT]) ||
274 (LastFlat[VM_CNT] > ScoreLBs[VM_CNT] &&
275 LastFlat[VM_CNT] <= ScoreUBs[VM_CNT]));
278 void setPendingFlat() {
285 bool hasOtherPendingVmemTypes(
int GprNo, VmemType V)
const {
286 assert(GprNo < NUM_ALL_VGPRS);
287 return VgprVmemTypes[GprNo] & ~(1 <<
V);
290 void clearVgprVmemTypes(
int GprNo) {
291 assert(GprNo < NUM_ALL_VGPRS);
292 VgprVmemTypes[GprNo] = 0;
305 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
306 unsigned OtherScore);
308 void setScoreLB(InstCounterType
T,
unsigned Val) {
313 void setScoreUB(InstCounterType
T,
unsigned Val) {
320 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
324 void setRegScore(
int GprNo, InstCounterType
T,
unsigned Val) {
325 if (GprNo < NUM_ALL_VGPRS) {
326 VgprUB = std::max(VgprUB, GprNo);
327 VgprScores[
T][GprNo] = Val;
330 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
331 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
337 unsigned OpNo,
unsigned Val);
340 HardwareLimits Limits = {};
341 RegisterEncoding Encoding = {};
342 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
343 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
344 unsigned PendingEvents = 0;
346 unsigned LastFlat[NUM_INST_CNTS] = {0};
351 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
353 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
356 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
374 std::unique_ptr<WaitcntBrackets> Incoming;
382 bool ForceEmitZeroWaitcnts;
383 bool ForceEmitWaitcnt[NUM_INST_CNTS];
395 (void)ForceExpCounter;
396 (void)ForceLgkmCounter;
397 (void)ForceVMCounter;
400 bool shouldFlushVmCnt(
MachineLoop *
ML, WaitcntBrackets &Brackets);
402 WaitcntBrackets &ScoreBrackets);
407 return "SI insert wait instructions";
417 bool isForceEmitWaitcnt()
const {
418 for (
auto T : inst_counter_types())
419 if (ForceEmitWaitcnt[
T])
424 void setForceEmitWaitcnt() {
430 ForceEmitWaitcnt[
EXP_CNT] =
true;
432 ForceEmitWaitcnt[
EXP_CNT] =
false;
444 ForceEmitWaitcnt[
VM_CNT] =
true;
446 ForceEmitWaitcnt[
VM_CNT] =
false;
453 WaitEventType getVmemWaitEventType(
const MachineInstr &Inst)
const {
461 return SCRATCH_WRITE_ACCESS;
462 return VMEM_WRITE_ACCESS;
464 return VMEM_READ_ACCESS;
471 WaitcntBrackets &ScoreBrackets,
475 WaitcntBrackets &ScoreBrackets,
482 WaitcntBrackets *ScoreBrackets);
484 WaitcntBrackets &ScoreBrackets);
485 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
493RegInterval WaitcntBrackets::getRegInterval(
const MachineInstr *
MI,
497 unsigned OpNo)
const {
499 if (!
TRI->isInAllocatableClass(
Op.getReg()))
511 if (
TRI->isVectorRegister(*
MRI,
Op.getReg())) {
512 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
515 Result.first += AGPR_OFFSET;
517 }
else if (
TRI->isSGPRReg(*
MRI,
Op.getReg())) {
518 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
519 Result.first =
Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
521 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
529 unsigned Size =
TRI->getRegSizeInBits(*RC);
541 assert(
TRI->isVectorRegister(*
MRI,
MI->getOperand(OpNo).getReg()));
543 setRegScore(RegNo, EXP_CNT, Val);
552 MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD;
559 InstCounterType
T = eventCounter(
E);
560 unsigned CurrScore = getScoreUB(
T) + 1;
566 PendingEvents |= 1 <<
E;
567 setScoreUB(
T, CurrScore);
577 if (AddrOpIdx != -1) {
578 setExpScore(&Inst,
TII,
TRI,
MRI, AddrOpIdx, CurrScore);
591 AMDGPU::OpName::data1),
596 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
597 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
600 if (
Op.isReg() && !
Op.isDef() &&
601 TRI->isVectorRegister(*
MRI,
Op.getReg())) {
602 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
606 }
else if (
TII->isFLAT(Inst)) {
618 }
else if (
TII->isMIMG(Inst)) {
620 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
627 }
else if (
TII->isMTBUF(Inst)) {
629 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
631 }
else if (
TII->isMUBUF(Inst)) {
633 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
640 }
else if (
TII->isLDSDIR(Inst)) {
647 if (
TII->isEXP(Inst)) {
666 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
671 }
else if (Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORD ||
672 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 ||
673 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) {
679 setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore);
686 if (!
Op.isReg() || !
Op.isDef())
690 if (
Interval.first >= NUM_ALL_VGPRS)
692 if (updateVMCntOnly(Inst)) {
697 VmemType
V = getVmemType(Inst);
699 VgprVmemTypes[RegNo] |= 1 <<
V;
703 setRegScore(RegNo,
T, CurrScore);
707 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS,
T, CurrScore);
714 for (
auto T : inst_counter_types()) {
715 unsigned SR = getScoreRange(
T);
719 OS <<
" VM_CNT(" << SR <<
"): ";
722 OS <<
" LGKM_CNT(" << SR <<
"): ";
725 OS <<
" EXP_CNT(" << SR <<
"): ";
728 OS <<
" VS_CNT(" << SR <<
"): ";
731 OS <<
" UNKNOWN(" << SR <<
"): ";
737 unsigned LB = getScoreLB(
T);
739 for (
int J = 0; J <= VgprUB; J++) {
740 unsigned RegScore = getRegScore(J,
T);
743 unsigned RelScore = RegScore - LB - 1;
744 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
745 OS << RelScore <<
":v" << J <<
" ";
747 OS << RelScore <<
":ds ";
752 for (
int J = 0; J <= SgprUB; J++) {
753 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS, LGKM_CNT);
756 unsigned RelScore = RegScore - LB - 1;
757 OS << RelScore <<
":s" << J <<
" ";
769 simplifyWaitcnt(VM_CNT,
Wait.VmCnt);
770 simplifyWaitcnt(EXP_CNT,
Wait.ExpCnt);
771 simplifyWaitcnt(LGKM_CNT,
Wait.LgkmCnt);
772 simplifyWaitcnt(VS_CNT,
Wait.VsCnt);
775void WaitcntBrackets::simplifyWaitcnt(InstCounterType
T,
776 unsigned &Count)
const {
780 if (Count >= getScoreRange(
T))
784void WaitcntBrackets::determineWait(InstCounterType
T,
int RegNo,
786 unsigned ScoreToWait = getRegScore(RegNo,
T);
790 const unsigned LB = getScoreLB(
T);
791 const unsigned UB = getScoreUB(
T);
792 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
793 if ((
T == VM_CNT ||
T == LGKM_CNT) &&
795 !
ST->hasFlatLgkmVMemCountInOrder()) {
800 }
else if (counterOutOfOrder(
T)) {
808 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(
T) - 1);
809 addWait(
Wait,
T, NeededWait);
815 applyWaitcnt(VM_CNT,
Wait.VmCnt);
816 applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
817 applyWaitcnt(LGKM_CNT,
Wait.LgkmCnt);
818 applyWaitcnt(VS_CNT,
Wait.VsCnt);
821void WaitcntBrackets::applyWaitcnt(InstCounterType
T,
unsigned Count) {
822 const unsigned UB = getScoreUB(
T);
826 if (counterOutOfOrder(
T))
828 setScoreLB(
T, std::max(getScoreLB(
T), UB - Count));
831 PendingEvents &= ~WaitEventMaskForInst[
T];
837bool WaitcntBrackets::counterOutOfOrder(InstCounterType
T)
const {
839 if (
T == LGKM_CNT && hasPendingEvent(SMEM_ACCESS))
841 return hasMixedPendingEvents(
T);
851char SIInsertWaitcnts::
ID = 0;
856 return new SIInsertWaitcnts();
866 if (NewEnc == MO.
getImm())
877bool SIInsertWaitcnts::applyPreexistingWaitcnt(
878 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
886 if (II.isMetaInstruction())
889 if (II.getOpcode() == AMDGPU::S_WAITCNT) {
893 if (!TrackedWaitcntSet.
count(&II)) {
908 assert(II.getOpcode() == AMDGPU::S_WAITCNT_VSCNT);
909 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
910 if (!TrackedWaitcntSet.
count(&II)) {
912 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
913 Wait.VsCnt = std::min(
Wait.VsCnt, OldVSCnt);
916 if (!WaitcntVsCntInstr) {
917 WaitcntVsCntInstr = &II;
927 if (
Wait.hasWaitExceptVsCnt()) {
931 ScoreBrackets.applyWaitcnt(
Wait);
937 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
938 <<
"New Instr at block end: " << *WaitcntInstr
940 :
dbgs() <<
"applyPreexistingWaitcnt\n"
941 <<
"Old Instr: " << *It
942 <<
"New Instr: " << *WaitcntInstr <<
'\n');
950 if (WaitcntVsCntInstr) {
951 if (
Wait.hasWaitVsCnt()) {
954 AMDGPU::OpName::simm16,
Wait.VsCnt);
955 ScoreBrackets.applyWaitcnt(
Wait);
959 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
960 <<
"New Instr at block end: "
961 << *WaitcntVsCntInstr <<
'\n'
962 :
dbgs() <<
"applyPreexistingWaitcnt\n"
963 <<
"Old Instr: " << *It
964 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
975 unsigned Opc =
MI.getOpcode();
976 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
977 !
MI.getOperand(1).isUndef();
1007bool SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &
MI,
1008 WaitcntBrackets &ScoreBrackets,
1011 setForceEmitWaitcnt();
1013 if (
MI.isMetaInstruction())
1022 if (
MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1023 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1024 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1025 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1026 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1033 if (
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1034 MI.getOpcode() == AMDGPU::SI_RETURN ||
1035 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1043 else if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1044 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1046 ScoreBrackets.getScoreRange(VS_CNT) != 0 &&
1047 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1051 else if ((
MI.getOpcode() == AMDGPU::S_SENDMSG ||
1052 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1053 ST->hasLegacyGeometry() &&
1059 else if (
MI.getOpcode() == SC_FENCE) {
1060 const unsigned int group_size =
1061 context->shader_info->GetMaxThreadGroupSize();
1063 const bool group_is_multi_wave =
1064 (group_size == 0 || group_size > target_info->GetWaveFrontSize());
1065 const bool fence_is_global = !((SCInstInternalMisc*)Inst)->IsGroupFence();
1067 for (
unsigned int i = 0; i < Inst->NumSrcOperands(); i++) {
1068 SCRegType src_type = Inst->GetSrcType(i);
1071 if (group_is_multi_wave ||
1072 context->OptFlagIsOn(OPT_R1100_LDSMEM_FENCE_CHICKEN_BIT)) {
1073 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT,
1074 ScoreBrackets->getScoreUB(LGKM_CNT));
1076 if (target_info->HasBufferLoadToLDS()) {
1077 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT,
1078 ScoreBrackets->getScoreUB(VM_CNT));
1084 if (group_is_multi_wave || fence_is_global) {
1085 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1086 ScoreBrackets->getScoreUB(EXP_CNT));
1087 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT,
1088 ScoreBrackets->getScoreUB(LGKM_CNT));
1096 if (group_is_multi_wave || fence_is_global) {
1097 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1098 ScoreBrackets->getScoreUB(EXP_CNT));
1099 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT,
1100 ScoreBrackets->getScoreUB(VM_CNT));
1117 if (
MI.modifiesRegister(AMDGPU::EXEC,
TRI)) {
1120 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1121 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1122 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1123 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1137 if (
MI.getOperand(CallAddrOpIdx).isReg()) {
1138 RegInterval CallAddrOpInterval =
1139 ScoreBrackets.getRegInterval(&
MI,
TII,
MRI,
TRI, CallAddrOpIdx);
1141 for (
int RegNo = CallAddrOpInterval.first;
1142 RegNo < CallAddrOpInterval.second; ++RegNo)
1143 ScoreBrackets.determineWait(LGKM_CNT, RegNo,
Wait);
1147 if (RtnAddrOpIdx != -1) {
1148 RegInterval RtnAddrOpInterval =
1149 ScoreBrackets.getRegInterval(&
MI,
TII,
MRI,
TRI, RtnAddrOpIdx);
1151 for (
int RegNo = RtnAddrOpInterval.first;
1152 RegNo < RtnAddrOpInterval.second; ++RegNo)
1153 ScoreBrackets.determineWait(LGKM_CNT, RegNo,
Wait);
1171 const Value *
Ptr = Memop->getValue();
1172 if (Memop->isStore() && SLoadAddresses.
count(
Ptr)) {
1173 addWait(
Wait, LGKM_CNT, 0);
1177 unsigned AS = Memop->getAddrSpace();
1183 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1185 ScoreBrackets.determineWait(VM_CNT, RegNo,
Wait);
1186 if (Memop->isStore()) {
1187 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1192 for (
unsigned I = 0,
E =
MI.getNumOperands();
I !=
E; ++
I) {
1198 if (
Op.isTied() &&
Op.isUse() &&
TII->doesNotReadTiedSource(
MI))
1204 const bool IsVGPR =
TRI->isVectorRegister(*
MRI,
Op.getReg());
1211 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
1212 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1214 ScoreBrackets.determineWait(VM_CNT, RegNo,
Wait);
1215 ScoreBrackets.clearVgprVmemTypes(RegNo);
1217 if (
Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1218 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1221 ScoreBrackets.determineWait(LGKM_CNT, RegNo,
Wait);
1231 if (
MI.getOpcode() == AMDGPU::S_BARRIER &&
1232 !
ST->hasAutoWaitcntBeforeBarrier() && !
ST->supportsBackOffBarrier()) {
1240 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1246 ScoreBrackets.simplifyWaitcnt(
Wait);
1248 if (ForceEmitZeroWaitcnts)
1251 if (ForceEmitWaitcnt[VM_CNT])
1253 if (ForceEmitWaitcnt[EXP_CNT])
1255 if (ForceEmitWaitcnt[LGKM_CNT])
1259 if (ScoreBrackets.hasPendingEvent(VM_CNT))
1263 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
1270 WaitcntBrackets &ScoreBrackets,
1274 if (!ScoreBrackets.hasPendingEvent(VM_CNT))
1279 return generateWaitcnt(
Wait,
Block.instr_end(),
Block, ScoreBrackets,
1286 WaitcntBrackets &ScoreBrackets,
1291 if (OldWaitcntInstr)
1295 applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
1297 ScoreBrackets.applyWaitcnt(
Wait);
1300 if (
Wait.ExpCnt != ~0u && It !=
Block.instr_end() &&
1303 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1311 <<
"Update Instr: " << *It);
1316 if (
Wait.hasWaitExceptVsCnt()) {
1320 TrackedWaitcntSet.
insert(SWaitInst);
1324 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1325 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1328 if (
Wait.hasWaitVsCnt()) {
1334 TrackedWaitcntSet.
insert(SWaitInst);
1338 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1339 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1347bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(
const MachineInstr &
MI)
const {
1355 if (
MI.memoperands_empty())
1364 unsigned AS = Memop->getAddrSpace();
1375bool SIInsertWaitcnts::mayAccessLDSThroughFlat(
const MachineInstr &
MI)
const {
1379 if (!
TII->usesLGKM_CNT(
MI))
1383 if (
ST->isTgSplitEnabled())
1388 if (
MI.memoperands_empty())
1393 unsigned AS = Memop->getAddrSpace();
1403bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
1408 if (
TII->isFLATScratch(
MI))
1412 if (
TII->isFLATGlobal(
MI))
1417 if (
MI.memoperands_empty())
1422 unsigned AS = Memop->getAddrSpace();
1423 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
1427void SIInsertWaitcnts::updateEventWaitcntAfter(
MachineInstr &Inst,
1428 WaitcntBrackets *ScoreBrackets) {
1433 if (
TII->isDS(Inst) &&
TII->usesLGKM_CNT(Inst)) {
1435 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
1436 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_ACCESS, Inst);
1437 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_GPR_LOCK, Inst);
1439 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
1441 }
else if (
TII->isFLAT(Inst)) {
1444 int FlatASCount = 0;
1446 if (mayAccessVMEMThroughFlat(Inst)) {
1448 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
1452 if (mayAccessLDSThroughFlat(Inst)) {
1454 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
1463 if (FlatASCount > 1)
1464 ScoreBrackets->setPendingFlat();
1467 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
1470 if (
ST->vmemWriteNeedsExpWaitcnt() &&
1472 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, VMW_GPR_LOCK, Inst);
1474 }
else if (
TII->isSMRD(Inst)) {
1475 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
1476 }
else if (Inst.
isCall()) {
1485 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_LDS_ACCESS, Inst);
1486 }
else if (
TII->isVINTERP(Inst)) {
1487 int64_t
Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
1488 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
1490 unsigned Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
1492 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_PARAM_ACCESS, Inst);
1494 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_POS_ACCESS, Inst);
1496 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_GPR_LOCK, Inst);
1499 case AMDGPU::S_SENDMSG:
1500 case AMDGPU::S_SENDMSG_RTN_B32:
1501 case AMDGPU::S_SENDMSG_RTN_B64:
1502 case AMDGPU::S_SENDMSGHALT:
1503 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SQ_MESSAGE, Inst);
1505 case AMDGPU::S_MEMTIME:
1506 case AMDGPU::S_MEMREALTIME:
1507 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
1513bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
1514 unsigned OtherScore) {
1515 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
1516 unsigned OtherShifted =
1517 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
1518 Score = std::max(MyShifted, OtherShifted);
1519 return OtherShifted > MyShifted;
1527bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
1528 bool StrictDom =
false;
1530 VgprUB = std::max(VgprUB,
Other.VgprUB);
1531 SgprUB = std::max(SgprUB,
Other.SgprUB);
1533 for (
auto T : inst_counter_types()) {
1535 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[
T];
1536 const unsigned OtherEvents =
Other.PendingEvents & WaitEventMaskForInst[
T];
1537 if (OtherEvents & ~OldEvents)
1539 PendingEvents |= OtherEvents;
1542 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
1543 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
1544 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
1545 if (NewUB < ScoreLBs[
T])
1549 M.OldLB = ScoreLBs[
T];
1550 M.OtherLB =
Other.ScoreLBs[
T];
1551 M.MyShift = NewUB - ScoreUBs[
T];
1552 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
1554 ScoreUBs[
T] = NewUB;
1556 StrictDom |= mergeScore(M, LastFlat[
T],
Other.LastFlat[
T]);
1558 for (
int J = 0; J <= VgprUB; J++)
1559 StrictDom |= mergeScore(M, VgprScores[
T][J],
Other.VgprScores[
T][J]);
1561 if (
T == LGKM_CNT) {
1562 for (
int J = 0; J <= SgprUB; J++)
1563 StrictDom |= mergeScore(M, SgprScores[J],
Other.SgprScores[J]);
1567 for (
int J = 0; J <= VgprUB; J++) {
1568 unsigned char NewVmemTypes = VgprVmemTypes[J] |
Other.VgprVmemTypes[J];
1569 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
1570 VgprVmemTypes[J] = NewVmemTypes;
1577 return Inst.
getOpcode() == AMDGPU::S_WAITCNT ||
1578 (Inst.
getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1586 WaitcntBrackets &ScoreBrackets) {
1590 dbgs() <<
"*** Block" <<
Block.getNumber() <<
" ***";
1591 ScoreBrackets.dump();
1597 bool VCCZCorrect =
true;
1598 if (
ST->hasReadVCCZBug()) {
1601 VCCZCorrect =
false;
1602 }
else if (!
ST->partialVCCWritesUpdateVCCZ()) {
1605 VCCZCorrect =
false;
1619 if (!OldWaitcntInstr)
1620 OldWaitcntInstr = &Inst;
1625 bool FlushVmCnt =
Block.getFirstTerminator() == Inst &&
1626 isPreheaderToFlush(
Block, ScoreBrackets);
1629 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
1631 OldWaitcntInstr =
nullptr;
1634 bool RestoreVCCZ = !VCCZCorrect &&
readsVCCZ(Inst);
1637 if (
ST->hasReadVCCZBug() || !
ST->partialVCCWritesUpdateVCCZ()) {
1641 if (!
ST->partialVCCWritesUpdateVCCZ())
1642 VCCZCorrect =
false;
1651 if (
ST->hasReadVCCZBug() &&
1652 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1655 VCCZCorrect =
false;
1663 if (
TII->isSMRD(Inst)) {
1667 if (!Memop->isInvariant()) {
1668 const Value *
Ptr = Memop->getValue();
1672 if (
ST->hasReadVCCZBug()) {
1674 VCCZCorrect =
false;
1678 updateEventWaitcntAfter(Inst, &ScoreBrackets);
1684 if (RequireCheckResourceType(Inst, context)) {
1686 ScoreBrackets->setScoreLB(VM_CNT,
1687 ScoreBrackets->getScoreUB(VM_CNT));
1693 ScoreBrackets.dump();
1703 TII->get(
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
1713 if (
Block.getFirstTerminator() ==
Block.end() &&
1714 isPreheaderToFlush(
Block, ScoreBrackets))
1715 Modified |= generateWaitcntBlockEnd(
Block, ScoreBrackets, OldWaitcntInstr);
1723 WaitcntBrackets &ScoreBrackets) {
1724 auto [Iterator, IsInserted] = PreheadersToFlush.
try_emplace(&
MBB,
false);
1726 return Iterator->second;
1737 shouldFlushVmCnt(
Loop, ScoreBrackets)) {
1738 Iterator->second =
true;
1745bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
1759 WaitcntBrackets &Brackets) {
1760 bool HasVMemLoad =
false;
1761 bool HasVMemStore =
false;
1762 bool UsesVgprLoadedOutside =
false;
1768 if (isVMEMOrFlatVMEM(
MI)) {
1772 HasVMemStore =
true;
1774 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++) {
1776 if (!
Op.isReg() || !
TRI->isVectorRegister(*
MRI,
Op.getReg()))
1789 if (Brackets.getRegScore(RegNo, VM_CNT) > Brackets.getScoreLB(VM_CNT)) {
1790 UsesVgprLoadedOutside =
true;
1796 else if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad() &&
Op.isDef())
1807 if (!
ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
1809 return HasVMemLoad && UsesVgprLoadedOutside;
1814 TII =
ST->getInstrInfo();
1815 TRI = &
TII->getRegisterInfo();
1819 MLI = &getAnalysis<MachineLoopInfo>();
1820 PDT = &getAnalysis<MachinePostDominatorTree>();
1823 for (
auto T : inst_counter_types())
1824 ForceEmitWaitcnt[
T] =
false;
1829 HardwareLimits Limits = {};
1833 Limits.VscntMax =
ST->hasVscnt() ? 63 : 0;
1835 unsigned NumVGPRsMax =
ST->getAddressableNumVGPRs();
1836 unsigned NumSGPRsMax =
ST->getAddressableNumSGPRs();
1837 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
1838 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
1840 RegisterEncoding Encoding = {};
1843 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
1846 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
1848 TrackedWaitcntSet.
clear();
1862 I !=
E && (
I->isPHI() ||
I->isMetaInstruction()); ++
I)
1874 std::unique_ptr<WaitcntBrackets> Brackets;
1879 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
1882 BlockInfo &BI = BII->second;
1888 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
1890 *Brackets = *BI.Incoming;
1893 Brackets = std::make_unique<WaitcntBrackets>(ST, Limits, Encoding);
1895 *Brackets = WaitcntBrackets(ST, Limits, Encoding);
1898 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
1901 if (Brackets->hasPendingEvent()) {
1902 BlockInfo *MoveBracketsToSucc =
nullptr;
1904 auto SuccBII = BlockInfos.
find(Succ);
1905 BlockInfo &SuccBI = SuccBII->second;
1906 if (!SuccBI.Incoming) {
1907 SuccBI.Dirty =
true;
1910 if (!MoveBracketsToSucc) {
1911 MoveBracketsToSucc = &SuccBI;
1913 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
1915 }
else if (SuccBI.Incoming->merge(*Brackets)) {
1916 SuccBI.Dirty =
true;
1921 if (MoveBracketsToSucc)
1922 MoveBracketsToSucc->Incoming = std::move(Brackets);
1927 if (
ST->hasScalarStores()) {
1929 bool HaveScalarStores =
false;
1933 if (!HaveScalarStores &&
TII->isScalarStore(
MI))
1934 HaveScalarStores =
true;
1936 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1937 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
1942 if (HaveScalarStores) {
1952 bool SeenDCacheWB =
false;
1956 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
1957 SeenDCacheWB =
true;
1958 else if (
TII->isScalarStore(*
I))
1959 SeenDCacheWB =
false;
1962 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
1963 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
1976 if (
ST->requiresNopBeforeDeallocVGPRs()) {
1984 ReleaseVGPRInsts.clear();
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static bool readsVCCZ(const MachineInstr &MI)
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
bool isEntryFunction() const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptNone() const
Do not optimize this function (-O0).
Interval Class - An Interval is a set of nodes defined such that every node in the interval has all o...
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Represents a single loop in the control flow graph.
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
unsigned getNumOperands() const
Retuns the total number of operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isEXP(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static bool isVINTERP(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
static bool isVALU(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Iterator for intrusive lists based on ilist_node.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getVmcntBitMask(const IsaVersion &Version)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Represents the counter values to wait for in an s_waitcnt instruction.
static Waitcnt allZero(bool HasVscnt)
static Waitcnt allZeroExceptVsCnt()
static constexpr bool is_iterable